You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2015/02/02 19:41:39 UTC
[01/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Repository: flink
Updated Branches:
refs/heads/master 563e54623 -> 3d9267eb3
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java b/flink-staging/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java
new file mode 100644
index 0000000..e1235b6
--- /dev/null
+++ b/flink-staging/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.tachyon;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.ExecutionEnvironmentFactory;
+import org.apache.flink.api.java.LocalEnvironment;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.examples.java.wordcount.WordCount;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import tachyon.client.InStream;
+import tachyon.client.OutStream;
+import tachyon.client.ReadType;
+import tachyon.client.TachyonFS;
+import tachyon.client.TachyonFile;
+import tachyon.client.WriteType;
+import tachyon.master.LocalTachyonCluster;
+
+import java.io.File;
+import java.io.StringWriter;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+public class TachyonFileSystemWrapperTest {
+ private static final long TACHYON_WORKER_CAPACITY = 1024 * 1024 * 32;
+ private static final String TACHYON_TEST_IN_FILE_NAME = "tachyontest";
+ private static final String TACHYON_TEST_OUT_FILE_NAME = "result";
+ private static final Path HADOOP_CONFIG_PATH;
+
+ static {
+ URL resource = TachyonFileSystemWrapperTest.class.getResource("/tachyonHadoopConf.xml");
+ File file = null;
+ try {
+ file = new File(resource.toURI());
+ } catch (URISyntaxException e) {
+ throw new RuntimeException("Unable to load req. res", e);
+ }
+ if(!file.exists()) {
+ throw new RuntimeException("Unable to load required resource");
+ }
+ HADOOP_CONFIG_PATH = new Path(file.getAbsolutePath());
+ }
+
+ private LocalTachyonCluster cluster;
+ private TachyonFS client;
+ private String input;
+ private String output;
+
+ @Before
+ public void startTachyon() {
+ try {
+ cluster = new LocalTachyonCluster(TACHYON_WORKER_CAPACITY);
+ cluster.start();
+ client = cluster.getClient();
+ int id = client.createFile("/" + TACHYON_TEST_IN_FILE_NAME, 1024 * 32);
+ Assert.assertNotEquals("Unable to create file", -1, id);
+
+ TachyonFile testFile = client.getFile(id);
+ Assert.assertNotNull(testFile);
+
+
+ OutStream outStream = testFile.getOutStream(WriteType.MUST_CACHE);
+ for(int i = 0; i < 10; i++) {
+ outStream.write("Hello Tachyon\n".getBytes());
+ }
+ outStream.close();
+ final String tachyonBase = "tachyon://" + cluster.getMasterHostname() + ":" + cluster.getMasterPort();
+ input = tachyonBase + "/" + TACHYON_TEST_IN_FILE_NAME;
+ output = tachyonBase + "/" + TACHYON_TEST_OUT_FILE_NAME;
+
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail("Test preparation failed with exception: "+e.getMessage());
+ }
+ }
+
+ @After
+ public void stopTachyon() {
+ try {
+ cluster.stop();
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail("Test teardown failed with exception: "+e.getMessage());
+ }
+ }
+ // Verify that Hadoop's FileSystem can load the TFS (Tachyon File System)
+ @Test
+ public void testHadoopLoadability() {
+ try {
+ Path tPath = new Path(input);
+ Configuration conf = new Configuration();
+ conf.addResource(HADOOP_CONFIG_PATH);
+ Assert.assertEquals("tachyon.hadoop.TFS", conf.get("fs.tachyon.impl", null));
+ tPath.getFileSystem(conf);
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail("Test failed with exception: "+e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void testTachyon() {
+ try {
+ org.apache.flink.configuration.Configuration addHDConfToFlinkConf = new org.apache.flink.configuration.Configuration();
+ addHDConfToFlinkConf.setString(ConfigConstants.HDFS_DEFAULT_CONFIG, HADOOP_CONFIG_PATH.toString());
+ GlobalConfiguration.includeConfiguration(addHDConfToFlinkConf);
+
+ new DopOneTestEnvironment(); // initialize DOP one
+
+ WordCount.main(new String[]{input, output});
+
+ // verify result
+ TachyonFile resultFile = client.getFile("/" + TACHYON_TEST_OUT_FILE_NAME);
+ Assert.assertNotNull("Result file has not been created", resultFile);
+ InStream inStream = resultFile.getInStream(ReadType.CACHE);
+ Assert.assertNotNull("Result file has not been created", inStream);
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(inStream, writer);
+ String resultString = writer.toString();
+
+ Assert.assertEquals("hello 10\n" +
+ "tachyon 10\n", resultString);
+
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail("Test failed with exception: "+e.getMessage());
+ }
+ }
+
+ // package visible
+ static final class DopOneTestEnvironment extends LocalEnvironment {
+ static {
+ initializeContextEnvironment(new ExecutionEnvironmentFactory() {
+ @Override
+ public ExecutionEnvironment createExecutionEnvironment() {
+ LocalEnvironment le = new LocalEnvironment();
+ le.setDegreeOfParallelism(1);
+ return le;
+ }
+ });
+ }
+ }
+
+}
[15/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
new file mode 100644
index 0000000..915ca30
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
@@ -0,0 +1,1471 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple10;
+import org.apache.flink.api.java.tuple.Tuple11;
+import org.apache.flink.api.java.tuple.Tuple12;
+import org.apache.flink.api.java.tuple.Tuple13;
+import org.apache.flink.api.java.tuple.Tuple14;
+import org.apache.flink.api.java.tuple.Tuple15;
+import org.apache.flink.api.java.tuple.Tuple16;
+import org.apache.flink.api.java.tuple.Tuple17;
+import org.apache.flink.api.java.tuple.Tuple18;
+import org.apache.flink.api.java.tuple.Tuple19;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple20;
+import org.apache.flink.api.java.tuple.Tuple21;
+import org.apache.flink.api.java.tuple.Tuple22;
+import org.apache.flink.api.java.tuple.Tuple23;
+import org.apache.flink.api.java.tuple.Tuple24;
+import org.apache.flink.api.java.tuple.Tuple25;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple6;
+import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.tuple.Tuple9;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.streaming.api.invokable.operator.ProjectInvokable;
+
+public class StreamProjection<IN> {
+
+ private DataStream<IN> dataStream;
+ private int[] fieldIndexes;
+ private TypeInformation<IN> inTypeInfo;
+
+ protected StreamProjection(DataStream<IN> dataStream, int[] fieldIndexes) {
+ this.dataStream = dataStream;
+ this.fieldIndexes = fieldIndexes;
+ this.inTypeInfo = dataStream.getType();
+ if (!inTypeInfo.isTupleType()) {
+ throw new RuntimeException("Only Tuple DataStreams can be projected");
+ }
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0> SingleOutputStreamOperator<Tuple1<T0>, ?> types(Class<T0> type0) {
+ Class<?>[] types = { type0 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple1<T0>> outType = (TypeInformation<Tuple1<T0>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream.transform("Projection", outType, new ProjectInvokable<IN, Tuple1<T0>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1> SingleOutputStreamOperator<Tuple2<T0, T1>, ?> types(Class<T0> type0,
+ Class<T1> type1) {
+ Class<?>[] types = { type0, type1 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple2<T0, T1>> outType = (TypeInformation<Tuple2<T0, T1>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream.transform("Projection", outType,
+ new ProjectInvokable<IN, Tuple2<T0, T1>>(fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2> SingleOutputStreamOperator<Tuple3<T0, T1, T2>, ?> types(Class<T0> type0,
+ Class<T1> type1, Class<T2> type2) {
+ Class<?>[] types = { type0, type1, type2 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple3<T0, T1, T2>> outType = (TypeInformation<Tuple3<T0, T1, T2>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream.transform("Projection", outType,
+ new ProjectInvokable<IN, Tuple3<T0, T1, T2>>(fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3> SingleOutputStreamOperator<Tuple4<T0, T1, T2, T3>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3) {
+ Class<?>[] types = { type0, type1, type2, type3 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple4<T0, T1, T2, T3>> outType = (TypeInformation<Tuple4<T0, T1, T2, T3>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream.transform("Projection", outType,
+ new ProjectInvokable<IN, Tuple4<T0, T1, T2, T3>>(fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4> SingleOutputStreamOperator<Tuple5<T0, T1, T2, T3, T4>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4) {
+ Class<?>[] types = { type0, type1, type2, type3, type4 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple5<T0, T1, T2, T3, T4>> outType = (TypeInformation<Tuple5<T0, T1, T2, T3, T4>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream.transform("Projection", outType,
+ new ProjectInvokable<IN, Tuple5<T0, T1, T2, T3, T4>>(fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5> SingleOutputStreamOperator<Tuple6<T0, T1, T2, T3, T4, T5>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple6<T0, T1, T2, T3, T4, T5>> outType = (TypeInformation<Tuple6<T0, T1, T2, T3, T4, T5>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream.transform("Projection", outType,
+ new ProjectInvokable<IN, Tuple6<T0, T1, T2, T3, T4, T5>>(fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6> SingleOutputStreamOperator<Tuple7<T0, T1, T2, T3, T4, T5, T6>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple7<T0, T1, T2, T3, T4, T5, T6>> outType = (TypeInformation<Tuple7<T0, T1, T2, T3, T4, T5, T6>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform("Projection", outType,
+ new ProjectInvokable<IN, Tuple7<T0, T1, T2, T3, T4, T5, T6>>(fieldIndexes,
+ outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7> SingleOutputStreamOperator<Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>> outType = (TypeInformation<Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream.transform("Projection", outType,
+ new ProjectInvokable<IN, Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>>(fieldIndexes,
+ outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8> SingleOutputStreamOperator<Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>> outType = (TypeInformation<Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream.transform("Projection", outType,
+ new ProjectInvokable<IN, Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>>(fieldIndexes,
+ outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9> SingleOutputStreamOperator<Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>> outType = (TypeInformation<Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream.transform("Projection", outType,
+ new ProjectInvokable<IN, Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10> SingleOutputStreamOperator<Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>> outType = (TypeInformation<Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream.transform("Projection", outType,
+ new ProjectInvokable<IN, Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11> SingleOutputStreamOperator<Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>> outType = (TypeInformation<Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12> SingleOutputStreamOperator<Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>> outType = (TypeInformation<Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13> SingleOutputStreamOperator<Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>> outType = (TypeInformation<Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14> SingleOutputStreamOperator<Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>> outType = (TypeInformation<Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @param type15
+ * The class of field '15' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15> SingleOutputStreamOperator<Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14, Class<T15> type15) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14, type15 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>> outType = (TypeInformation<Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @param type15
+ * The class of field '15' of the result Tuples.
+ * @param type16
+ * The class of field '16' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16> SingleOutputStreamOperator<Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14, Class<T15> type15, Class<T16> type16) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14, type15, type16 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>> outType = (TypeInformation<Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @param type15
+ * The class of field '15' of the result Tuples.
+ * @param type16
+ * The class of field '16' of the result Tuples.
+ * @param type17
+ * The class of field '17' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17> SingleOutputStreamOperator<Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14, type15, type16, type17 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>> outType = (TypeInformation<Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @param type15
+ * The class of field '15' of the result Tuples.
+ * @param type16
+ * The class of field '16' of the result Tuples.
+ * @param type17
+ * The class of field '17' of the result Tuples.
+ * @param type18
+ * The class of field '18' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18> SingleOutputStreamOperator<Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
+ Class<T18> type18) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14, type15, type16, type17, type18 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>> outType = (TypeInformation<Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @param type15
+ * The class of field '15' of the result Tuples.
+ * @param type16
+ * The class of field '16' of the result Tuples.
+ * @param type17
+ * The class of field '17' of the result Tuples.
+ * @param type18
+ * The class of field '18' of the result Tuples.
+ * @param type19
+ * The class of field '19' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19> SingleOutputStreamOperator<Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
+ Class<T18> type18, Class<T19> type19) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14, type15, type16, type17, type18, type19 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>> outType = (TypeInformation<Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @param type15
+ * The class of field '15' of the result Tuples.
+ * @param type16
+ * The class of field '16' of the result Tuples.
+ * @param type17
+ * The class of field '17' of the result Tuples.
+ * @param type18
+ * The class of field '18' of the result Tuples.
+ * @param type19
+ * The class of field '19' of the result Tuples.
+ * @param type20
+ * The class of field '20' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20> SingleOutputStreamOperator<Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
+ Class<T18> type18, Class<T19> type19, Class<T20> type20) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14, type15, type16, type17, type18, type19,
+ type20 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>> outType = (TypeInformation<Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @param type15
+ * The class of field '15' of the result Tuples.
+ * @param type16
+ * The class of field '16' of the result Tuples.
+ * @param type17
+ * The class of field '17' of the result Tuples.
+ * @param type18
+ * The class of field '18' of the result Tuples.
+ * @param type19
+ * The class of field '19' of the result Tuples.
+ * @param type20
+ * The class of field '20' of the result Tuples.
+ * @param type21
+ * The class of field '21' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21> SingleOutputStreamOperator<Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
+ Class<T18> type18, Class<T19> type19, Class<T20> type20, Class<T21> type21) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14, type15, type16, type17, type18, type19,
+ type20, type21 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>> outType = (TypeInformation<Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @param type15
+ * The class of field '15' of the result Tuples.
+ * @param type16
+ * The class of field '16' of the result Tuples.
+ * @param type17
+ * The class of field '17' of the result Tuples.
+ * @param type18
+ * The class of field '18' of the result Tuples.
+ * @param type19
+ * The class of field '19' of the result Tuples.
+ * @param type20
+ * The class of field '20' of the result Tuples.
+ * @param type21
+ * The class of field '21' of the result Tuples.
+ * @param type22
+ * The class of field '22' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22> SingleOutputStreamOperator<Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
+ Class<T18> type18, Class<T19> type19, Class<T20> type20, Class<T21> type21,
+ Class<T22> type22) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14, type15, type16, type17, type18, type19,
+ type20, type21, type22 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>> outType = (TypeInformation<Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @param type15
+ * The class of field '15' of the result Tuples.
+ * @param type16
+ * The class of field '16' of the result Tuples.
+ * @param type17
+ * The class of field '17' of the result Tuples.
+ * @param type18
+ * The class of field '18' of the result Tuples.
+ * @param type19
+ * The class of field '19' of the result Tuples.
+ * @param type20
+ * The class of field '20' of the result Tuples.
+ * @param type21
+ * The class of field '21' of the result Tuples.
+ * @param type22
+ * The class of field '22' of the result Tuples.
+ * @param type23
+ * The class of field '23' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23> SingleOutputStreamOperator<Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
+ Class<T18> type18, Class<T19> type19, Class<T20> type20, Class<T21> type21,
+ Class<T22> type22, Class<T23> type23) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14, type15, type16, type17, type18, type19,
+ type20, type21, type22, type23 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>> outType = (TypeInformation<Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+ outType,
+ new ProjectInvokable<IN, Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>>(
+ fieldIndexes, outType));
+ }
+
+ /**
+ * Projects a {@link Tuple} {@link DataStream} to the previously selected
+ * fields. Requires the classes of the fields of the resulting Tuples.
+ *
+ * @param type0
+ * The class of field '0' of the result Tuples.
+ * @param type1
+ * The class of field '1' of the result Tuples.
+ * @param type2
+ * The class of field '2' of the result Tuples.
+ * @param type3
+ * The class of field '3' of the result Tuples.
+ * @param type4
+ * The class of field '4' of the result Tuples.
+ * @param type5
+ * The class of field '5' of the result Tuples.
+ * @param type6
+ * The class of field '6' of the result Tuples.
+ * @param type7
+ * The class of field '7' of the result Tuples.
+ * @param type8
+ * The class of field '8' of the result Tuples.
+ * @param type9
+ * The class of field '9' of the result Tuples.
+ * @param type10
+ * The class of field '10' of the result Tuples.
+ * @param type11
+ * The class of field '11' of the result Tuples.
+ * @param type12
+ * The class of field '12' of the result Tuples.
+ * @param type13
+ * The class of field '13' of the result Tuples.
+ * @param type14
+ * The class of field '14' of the result Tuples.
+ * @param type15
+ * The class of field '15' of the result Tuples.
+ * @param type16
+ * The class of field '16' of the result Tuples.
+ * @param type17
+ * The class of field '17' of the result Tuples.
+ * @param type18
+ * The class of field '18' of the result Tuples.
+ * @param type19
+ * The class of field '19' of the result Tuples.
+ * @param type20
+ * The class of field '20' of the result Tuples.
+ * @param type21
+ * The class of field '21' of the result Tuples.
+ * @param type22
+ * The class of field '22' of the result Tuples.
+ * @param type23
+ * The class of field '23' of the result Tuples.
+ * @param type24
+ * The class of field '24' of the result Tuples.
+ * @return The projected DataStream.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24> SingleOutputStreamOperator<Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>, ?> types(
+ Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
+ Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
+ Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
+ Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
+ Class<T18> type18, Class<T19> type19, Class<T20> type20, Class<T21> type21,
+ Class<T22> type22, Class<T23> type23, Class<T24> type24) {
+ Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
+ type10, type11, type12, type13, type14, type15, type16, type17, type18, type19,
+ type20, type21, type22, type23, type24 };
+ if (types.length != this.fieldIndexes.length) {
+ throw new IllegalArgumentException(
+ "Numbers of projected fields and types do not match.");
+ }
+
+ @SuppressWarnings("unchecked")
+ TypeInformation<Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>> outType = (TypeInformation<Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>>) extractFieldTypes(
+ fieldIndexes, types, inTypeInfo);
+ return dataStream
+ .transform(
+ "Projection",
+
+ outType,
+ new ProjectInvokable<IN, Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>>(
+ fieldIndexes, outType));
+ }
+
+ public static TypeInformation<?> extractFieldTypes(int[] fields, Class<?>[] givenTypes,
+ TypeInformation<?> inType) {
+
+ TupleTypeInfo<?> inTupleType = (TupleTypeInfo<?>) inType;
+ TypeInformation<?>[] fieldTypes = new TypeInformation[fields.length];
+
+ for (int i = 0; i < fields.length; i++) {
+
+ if (inTupleType.getTypeAt(fields[i]).getTypeClass() != givenTypes[i]) {
+ throw new IllegalArgumentException(
+ "Given types do not match types of input data set.");
+ }
+
+ fieldTypes[i] = inTupleType.getTypeAt(fields[i]);
+ }
+
+ return new TupleTypeInfo<Tuple>(fieldTypes);
+ }
+
+}
[52/52] flink git commit: [FLINK-1452] Add 'flink-contrib' module
Posted by rm...@apache.org.
[FLINK-1452] Add 'flink-contrib' module
This closes #355
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3d9267eb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3d9267eb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3d9267eb
Branch: refs/heads/master
Commit: 3d9267eb3020b4671c67302cbfbf57291eb5a9bb
Parents: f94112f
Author: Robert Metzger <rm...@apache.org>
Authored: Fri Jan 30 15:29:05 2015 +0100
Committer: Robert Metzger <rm...@apache.org>
Committed: Mon Feb 2 19:41:05 2015 +0100
----------------------------------------------------------------------
flink-contrib/pom.xml | 41 ++++++++++++++++++++
.../main/java/org/apache/flink/contrib/.hidden | 0
flink-contrib/src/main/resources/.hidden | 0
.../test/java/org/apache/flink/contrib/.hidden | 0
pom.xml | 1 +
5 files changed, 42 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/3d9267eb/flink-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/flink-contrib/pom.xml b/flink-contrib/pom.xml
new file mode 100644
index 0000000..f0bfa01
--- /dev/null
+++ b/flink-contrib/pom.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-contrib</artifactId>
+ <name>flink-contrib</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/3d9267eb/flink-contrib/src/main/java/org/apache/flink/contrib/.hidden
----------------------------------------------------------------------
diff --git a/flink-contrib/src/main/java/org/apache/flink/contrib/.hidden b/flink-contrib/src/main/java/org/apache/flink/contrib/.hidden
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/flink/blob/3d9267eb/flink-contrib/src/main/resources/.hidden
----------------------------------------------------------------------
diff --git a/flink-contrib/src/main/resources/.hidden b/flink-contrib/src/main/resources/.hidden
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/flink/blob/3d9267eb/flink-contrib/src/test/java/org/apache/flink/contrib/.hidden
----------------------------------------------------------------------
diff --git a/flink-contrib/src/test/java/org/apache/flink/contrib/.hidden b/flink-contrib/src/test/java/org/apache/flink/contrib/.hidden
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/flink/blob/3d9267eb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4f3df5b..feb7837 100644
--- a/pom.xml
+++ b/pom.xml
@@ -64,6 +64,7 @@ under the License.
<module>flink-test-utils</module>
<module>flink-staging</module>
<module>flink-quickstart</module>
+ <module>flink-contrib</module>
<module>flink-dist</module>
</modules>
[22/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceFunctionITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceFunctionITCase.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceFunctionITCase.java
new file mode 100644
index 0000000..1801c3e
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceFunctionITCase.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.hadoopcompatibility.mapred;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.hadoopcompatibility.mapred.HadoopReduceFunction;
+import org.apache.flink.test.util.MultipleProgramsTestBase;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class HadoopReduceFunctionITCase extends MultipleProgramsTestBase {
+
+ public HadoopReduceFunctionITCase(ExecutionMode mode){
+ super(mode);
+ }
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ @Test
+ public void testStandardGrouping() throws Exception{
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env).
+ map(new Mapper1());
+
+ DataSet<Tuple2<IntWritable, IntWritable>> commentCnts = ds.
+ groupBy(0).
+ reduceGroup(new HadoopReduceFunction<IntWritable, Text, IntWritable, IntWritable>(new CommentCntReducer()));
+
+ String resultPath = tempFolder.newFile().toURI().toString();
+
+ commentCnts.writeAsText(resultPath);
+ env.execute();
+
+ String expected = "(0,0)\n"+
+ "(1,3)\n" +
+ "(2,5)\n" +
+ "(3,5)\n" +
+ "(4,2)\n";
+
+ compareResultsByLinesInMemory(expected, resultPath);
+ }
+
+ @Test
+ public void testUngroupedHadoopReducer() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
+
+ DataSet<Tuple2<IntWritable, IntWritable>> commentCnts = ds.
+ reduceGroup(new HadoopReduceFunction<IntWritable, Text, IntWritable, IntWritable>(new AllCommentCntReducer()));
+
+ String resultPath = tempFolder.newFile().toURI().toString();
+
+ commentCnts.writeAsText(resultPath);
+ env.execute();
+
+ String expected = "(42,15)\n";
+
+ compareResultsByLinesInMemory(expected, resultPath);
+ }
+
+ @Test
+ public void testConfigurationViaJobConf() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ JobConf conf = new JobConf();
+ conf.set("my.cntPrefix", "Hello");
+
+ DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env).
+ map(new Mapper2());
+
+ DataSet<Tuple2<IntWritable, IntWritable>> helloCnts = ds.
+ groupBy(0).
+ reduceGroup(new HadoopReduceFunction<IntWritable, Text, IntWritable, IntWritable>(
+ new ConfigurableCntReducer(), conf));
+
+ String resultPath = tempFolder.newFile().toURI().toString();
+
+ helloCnts.writeAsText(resultPath);
+ env.execute();
+
+ String expected = "(0,0)\n"+
+ "(1,0)\n" +
+ "(2,1)\n" +
+ "(3,1)\n" +
+ "(4,1)\n";
+
+ compareResultsByLinesInMemory(expected, resultPath);
+ }
+
+ public static class CommentCntReducer implements Reducer<IntWritable, Text, IntWritable, IntWritable> {
+
+ @Override
+ public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r)
+ throws IOException {
+ int commentCnt = 0;
+ while(vs.hasNext()) {
+ String v = vs.next().toString();
+ if(v.startsWith("Comment")) {
+ commentCnt++;
+ }
+ }
+ out.collect(k, new IntWritable(commentCnt));
+ }
+
+ @Override
+ public void configure(final JobConf arg0) { }
+
+ @Override
+ public void close() throws IOException { }
+ }
+
+ public static class AllCommentCntReducer implements Reducer<IntWritable, Text, IntWritable, IntWritable> {
+
+ @Override
+ public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r)
+ throws IOException {
+ int commentCnt = 0;
+ while(vs.hasNext()) {
+ String v = vs.next().toString();
+ if(v.startsWith("Comment")) {
+ commentCnt++;
+ }
+ }
+ out.collect(new IntWritable(42), new IntWritable(commentCnt));
+ }
+
+ @Override
+ public void configure(final JobConf arg0) { }
+
+ @Override
+ public void close() throws IOException { }
+ }
+
+ public static class ConfigurableCntReducer implements Reducer<IntWritable, Text, IntWritable, IntWritable> {
+ private String countPrefix;
+
+ @Override
+ public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r)
+ throws IOException {
+ int commentCnt = 0;
+ while(vs.hasNext()) {
+ String v = vs.next().toString();
+ if(v.startsWith(this.countPrefix)) {
+ commentCnt++;
+ }
+ }
+ out.collect(k, new IntWritable(commentCnt));
+ }
+
+ @Override
+ public void configure(final JobConf c) {
+ this.countPrefix = c.get("my.cntPrefix");
+ }
+
+ @Override
+ public void close() throws IOException { }
+ }
+
+ public static class Mapper1 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable, Text>> {
+ private static final long serialVersionUID = 1L;
+ @Override
+ public Tuple2<IntWritable, Text> map(Tuple2<IntWritable, Text> v)
+ throws Exception {
+ v.f0 = new IntWritable(v.f0.get() / 5);
+ return v;
+ }
+ }
+
+ public static class Mapper2 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable, Text>> {
+ private static final long serialVersionUID = 1L;
+ @Override
+ public Tuple2<IntWritable, Text> map(Tuple2<IntWritable, Text> v)
+ throws Exception {
+ v.f0 = new IntWritable(v.f0.get() % 5);
+ return v;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopTestData.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopTestData.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopTestData.java
new file mode 100644
index 0000000..eed6f8f
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopTestData.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.hadoopcompatibility.mapred;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+
+public class HadoopTestData {
+
+ public static DataSet<Tuple2<IntWritable, Text>> getKVPairDataSet(ExecutionEnvironment env) {
+
+ List<Tuple2<IntWritable, Text>> data = new ArrayList<Tuple2<IntWritable, Text>>();
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(1),new Text("Hi")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(2),new Text("Hello")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(3),new Text("Hello world")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(4),new Text("Hello world, how are you?")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(5),new Text("I am fine.")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(6),new Text("Luke Skywalker")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(7),new Text("Comment#1")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(8),new Text("Comment#2")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(9),new Text("Comment#3")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(10),new Text("Comment#4")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(11),new Text("Comment#5")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(12),new Text("Comment#6")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(13),new Text("Comment#7")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(14),new Text("Comment#8")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(15),new Text("Comment#9")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(16),new Text("Comment#10")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(17),new Text("Comment#11")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(18),new Text("Comment#12")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(19),new Text("Comment#13")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(20),new Text("Comment#14")));
+ data.add(new Tuple2<IntWritable, Text>(new IntWritable(21),new Text("Comment#15")));
+
+ Collections.shuffle(data);
+
+ return env.fromCollection(data);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/record/HadoopRecordInputOutputITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/record/HadoopRecordInputOutputITCase.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/record/HadoopRecordInputOutputITCase.java
new file mode 100644
index 0000000..fe7ea8e
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/record/HadoopRecordInputOutputITCase.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.test.hadoopcompatibility.mapred.record;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.hadoopcompatibility.mapred.record.example.WordCountWithOutputFormat;
+import org.apache.flink.test.testdata.WordCountData;
+import org.apache.flink.test.util.RecordAPITestBase;
+
+/**
+ * test the hadoop inputformat and outputformat
+ */
+public class HadoopRecordInputOutputITCase extends RecordAPITestBase {
+ protected String textPath;
+ protected String resultPath;
+ protected String counts;
+
+ @Override
+ protected void preSubmit() throws Exception {
+ textPath = createTempFile("text.txt", WordCountData.TEXT);
+ resultPath = getTempDirPath("result");
+ counts = WordCountData.COUNTS.replaceAll(" ", "\t");
+ }
+
+ @Override
+ protected Plan getTestJob() {
+ //WordCountWithHadoopOutputFormat takes hadoop TextInputFormat as input and output file in hadoop TextOutputFormat
+ WordCountWithOutputFormat wc = new WordCountWithOutputFormat();
+ return wc.getPlan("1", textPath, resultPath);
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ // Test results, append /1 to resultPath due to the generated _temproray file.
+ compareResultsByLinesInMemory(counts, resultPath + "/1");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIteratorTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIteratorTest.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIteratorTest.java
new file mode 100644
index 0000000..2592b88
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIteratorTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.hadoopcompatibility.mapred.wrapper;
+
+import java.util.ArrayList;
+import java.util.NoSuchElementException;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopTupleUnwrappingIterator;
+import org.apache.hadoop.io.IntWritable;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class HadoopTupleUnwrappingIteratorTest {
+
+ @Test
+ public void testValueIterator() {
+
+ HadoopTupleUnwrappingIterator<IntWritable, IntWritable> valIt =
+ new HadoopTupleUnwrappingIterator<IntWritable, IntWritable>(IntWritable.class);
+
+ // many values
+
+ ArrayList<Tuple2<IntWritable, IntWritable>> tList = new ArrayList<Tuple2<IntWritable, IntWritable>>();
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(1)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(2)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(3)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(4)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(5)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(6)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(7)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(8)));
+
+ int expectedKey = 1;
+ int[] expectedValues = new int[] {1,2,3,4,5,6,7,8};
+
+ valIt.set(tList.iterator());
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+ for(int expectedValue : expectedValues) {
+ Assert.assertTrue(valIt.hasNext());
+ Assert.assertTrue(valIt.hasNext());
+ Assert.assertTrue(valIt.next().get() == expectedValue);
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+ }
+ Assert.assertFalse(valIt.hasNext());
+ Assert.assertFalse(valIt.hasNext());
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+
+ // one value
+
+ tList.clear();
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(2),new IntWritable(10)));
+
+ expectedKey = 2;
+ expectedValues = new int[]{10};
+
+ valIt.set(tList.iterator());
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+ for(int expectedValue : expectedValues) {
+ Assert.assertTrue(valIt.hasNext());
+ Assert.assertTrue(valIt.hasNext());
+ Assert.assertTrue(valIt.next().get() == expectedValue);
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+ }
+ Assert.assertFalse(valIt.hasNext());
+ Assert.assertFalse(valIt.hasNext());
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+
+ // more values
+
+ tList.clear();
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(3),new IntWritable(10)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(3),new IntWritable(4)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(3),new IntWritable(7)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(3),new IntWritable(9)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(21)));
+
+ expectedKey = 3;
+ expectedValues = new int[]{10,4,7,9,21};
+
+ valIt.set(tList.iterator());
+ Assert.assertTrue(valIt.hasNext());
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+ for(int expectedValue : expectedValues) {
+ Assert.assertTrue(valIt.hasNext());
+ Assert.assertTrue(valIt.hasNext());
+ Assert.assertTrue(valIt.next().get() == expectedValue);
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+ }
+ Assert.assertFalse(valIt.hasNext());
+ Assert.assertFalse(valIt.hasNext());
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+
+ // no has next calls
+
+ tList.clear();
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(5)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(8)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(42)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(-1)));
+ tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(0)));
+
+ expectedKey = 4;
+ expectedValues = new int[]{5,8,42,-1,0};
+
+ valIt.set(tList.iterator());
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+ for(int expectedValue : expectedValues) {
+ Assert.assertTrue(valIt.next().get() == expectedValue);
+ }
+ try {
+ valIt.next();
+ Assert.fail();
+ } catch (NoSuchElementException nsee) {
+ // expected
+ }
+ Assert.assertFalse(valIt.hasNext());
+ Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputFormatTest.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputFormatTest.java
new file mode 100644
index 0000000..d79afaa
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputFormatTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.test.hadoopcompatibility.mapreduce;
+
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.hadoopcompatibility.mapreduce.HadoopInputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.fail;
+
+
+
+public class HadoopInputFormatTest {
+
+
+ public class DummyVoidKeyInputFormat<T> extends FileInputFormat<Void, T> {
+
+ public DummyVoidKeyInputFormat() {
+ }
+
+ @Override
+ public RecordReader<Void, T> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
+ return null;
+ }
+ }
+
+
+ @Test
+ public void checkTypeInformation() {
+ try {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ // Set up the Hadoop Input Format
+ Job job = Job.getInstance();
+ HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<Void, Long>( new DummyVoidKeyInputFormat(), Void.class, Long.class, job);
+
+ TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
+ TypeInformation<Tuple2<Void,Long>> testTupleType = new TupleTypeInfo<Tuple2<Void,Long>>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
+
+ if(tupleType.isTupleType()) {
+ if(!((TupleTypeInfo)tupleType).equals(testTupleType)) {
+ fail("Tuple type information was not set correctly!");
+ }
+ } else {
+ fail("Type information was not set to tuple type information!");
+ }
+
+ }
+ catch (Exception ex) {
+ fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputOutputITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputOutputITCase.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputOutputITCase.java
new file mode 100644
index 0000000..7eee629
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputOutputITCase.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.hadoopcompatibility.mapreduce;
+
+import org.apache.flink.hadoopcompatibility.mapreduce.example.WordCount;
+import org.apache.flink.test.testdata.WordCountData;
+import org.apache.flink.test.util.JavaProgramTestBase;
+
+public class HadoopInputOutputITCase extends JavaProgramTestBase {
+
+ protected String textPath;
+ protected String resultPath;
+
+
+ @Override
+ protected void preSubmit() throws Exception {
+ textPath = createTempFile("text.txt", WordCountData.TEXT);
+ resultPath = getTempDirPath("result");
+ this.setDegreeOfParallelism(4);
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ compareResultsByLinesInMemory(WordCountData.COUNTS, resultPath, new String[]{".", "_"});
+ }
+
+ @Override
+ protected void testProgram() throws Exception {
+ WordCount.main(new String[] { textPath, resultPath });
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/resources/log4j-test.properties b/flink-staging/flink-hadoop-compatibility/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..0b686e5
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/resources/log4j-test.properties
@@ -0,0 +1,27 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=OFF, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/resources/logback-test.xml b/flink-staging/flink-hadoop-compatibility/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..8b3bb27
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/resources/logback-test.xml
@@ -0,0 +1,29 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hbase/pom.xml b/flink-staging/flink-hbase/pom.xml
new file mode 100644
index 0000000..223389c
--- /dev/null
+++ b/flink-staging/flink-hbase/pom.xml
@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-staging</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-hbase</artifactId>
+ <name>flink-hbase</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <hbase.hadoop1.version>0.98.6.1-hadoop1</hbase.hadoop1.version>
+ <hbase.hadoop2.version>0.98.6.1-hadoop2</hbase.hadoop2.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-hadoop-compatibility</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.version}</version>
+ <exclusions>
+ <!-- Remove unneeded dependency, which is conflicting with our jetty-util version. -->
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>hadoop-1</id>
+ <activation>
+ <property>
+ <!-- Please do not remove the 'hadoop1' comment. See ./tools/generate_specific_pom.sh -->
+ <!--hadoop1--><name>hadoop.profile</name><value>1</value>
+ </property>
+ </activation>
+ <properties>
+ <hbase.version>${hbase.hadoop1.version}</hbase.version>
+ </properties>
+ <dependencies>
+ <!-- Force hadoop-common dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hadoop-2</id>
+ <activation>
+ <property>
+ <!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh -->
+ <!--hadoop2--><name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <properties>
+ <hbase.version>${hbase.hadoop2.version}</hbase.version>
+ </properties>
+ <dependencies>
+ <!-- Force hadoop-common dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>cdh5.1.3</id>
+ <properties>
+ <hadoop.profile>2</hadoop.profile>
+ <hbase.version>0.98.1-cdh5.1.3</hbase.version>
+ <hadoop.version>2.3.0-cdh5.1.3</hadoop.version>
+ <!-- Cloudera use different versions for hadoop core and commons-->
+ <!-- This profile could be removed if Cloudera fix this mismatch! -->
+ <hadoop.core.version>2.3.0-mr1-cdh5.1.3</hadoop.core.version>
+ </properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <version>${hadoop.core.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ <dependencies>
+ <!-- Force hadoop-common dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputFormat.java b/flink-staging/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputFormat.java
new file mode 100755
index 0000000..9c861ed
--- /dev/null
+++ b/flink-staging/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputFormat.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.addons.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.io.LocatableInputSplitAssigner;
+import org.apache.flink.api.common.io.statistics.BaseStatistics;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.io.InputSplitAssigner;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link InputFormat} subclass that wraps the access for HTables.
+ *
+ */
+public abstract class TableInputFormat<T extends Tuple> implements InputFormat<T, TableInputSplit>{
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(TableInputFormat.class);
+
+ /** helper variable to decide whether the input is exhausted or not */
+ private boolean endReached = false;
+
+ // TODO table and scan could be serialized when kryo serializer will be the default
+ protected transient HTable table;
+ protected transient Scan scan;
+
+ /** HBase iterator wrapper */
+ private ResultScanner rs;
+
+ private byte[] lastRow;
+ private int scannedRows;
+
+ // abstract methods allow for multiple table and scanners in the same job
+ protected abstract Scan getScanner();
+ protected abstract String getTableName();
+ protected abstract T mapResultToTuple(Result r);
+
+ /**
+ * creates a {@link Scan} object and a {@link HTable} connection
+ *
+ * @param parameters
+ * @see Configuration
+ */
+ @Override
+ public void configure(Configuration parameters) {
+ this.table = createTable();
+ this.scan = getScanner();
+ }
+
+ /** Create an {@link HTable} instance and set it into this format */
+ private HTable createTable() {
+ LOG.info("Initializing HBaseConfiguration");
+ //use files found in the classpath
+ org.apache.hadoop.conf.Configuration hConf = HBaseConfiguration.create();
+
+ try {
+ return new HTable(hConf, getTableName());
+ } catch (Exception e) {
+ LOG.error("Error instantiating a new HTable instance", e);
+ }
+ return null;
+ }
+
+ @Override
+ public boolean reachedEnd() throws IOException {
+ return this.endReached;
+ }
+
+ @Override
+ public T nextRecord(T reuse) throws IOException {
+ if (this.rs == null){
+ throw new IOException("No table result scanner provided!");
+ }
+ try{
+ Result res = this.rs.next();
+ if (res != null){
+ scannedRows++;
+ lastRow = res.getRow();
+ return mapResultToTuple(res);
+ }
+ }catch (Exception e) {
+ this.rs.close();
+ //workaround for timeout on scan
+ StringBuffer logMsg = new StringBuffer("Error after scan of ")
+ .append(scannedRows)
+ .append(" rows. Retry with a new scanner...");
+ LOG.warn(logMsg.toString(), e);
+ this.scan.setStartRow(lastRow);
+ this.rs = table.getScanner(scan);
+ Result res = this.rs.next();
+ if (res != null) {
+ scannedRows++;
+ lastRow = res.getRow();
+ return mapResultToTuple(res);
+ }
+ }
+
+ this.endReached = true;
+ return null;
+ }
+
+ @Override
+ public void open(TableInputSplit split) throws IOException {
+ if (split == null){
+ throw new IOException("Input split is null!");
+ }
+ if (table == null){
+ throw new IOException("No HTable provided!");
+ }
+ if (scan == null){
+ throw new IOException("No Scan instance provided");
+ }
+
+ logSplitInfo("opening", split);
+ scan.setStartRow(split.getStartRow());
+ lastRow = split.getEndRow();
+ scan.setStopRow(lastRow);
+
+ this.rs = table.getScanner(scan);
+ this.endReached = false;
+ this.scannedRows = 0;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if(rs!=null){
+ this.rs.close();
+ }
+ if(table!=null){
+ this.table.close();
+ }
+ LOG.info("Closing split (scanned {} rows)", scannedRows);
+ this.lastRow = null;
+ }
+
+ @Override
+ public TableInputSplit[] createInputSplits(final int minNumSplits) throws IOException {
+ //Gets the starting and ending row keys for every region in the currently open table
+ final Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
+ if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) {
+ throw new IOException("Expecting at least one region.");
+ }
+ final byte[] startRow = scan.getStartRow();
+ final byte[] stopRow = scan.getStopRow();
+ final boolean scanWithNoLowerBound = startRow.length == 0;
+ final boolean scanWithNoUpperBound = stopRow.length == 0;
+
+ final List<TableInputSplit> splits = new ArrayList<TableInputSplit>(minNumSplits);
+ for (int i = 0; i < keys.getFirst().length; i++) {
+ final byte[] startKey = keys.getFirst()[i];
+ final byte[] endKey = keys.getSecond()[i];
+ final String regionLocation = table.getRegionLocation(startKey, false).getHostnamePort();
+ //Test if the given region is to be included in the InputSplit while splitting the regions of a table
+ if (!includeRegionInSplit(startKey, endKey)) {
+ continue;
+ }
+ //Finds the region on which the given row is being served
+ final String[] hosts = new String[] { regionLocation };
+
+ // determine if regions contains keys used by the scan
+ boolean isLastRegion = endKey.length == 0;
+ if ((scanWithNoLowerBound || isLastRegion || Bytes.compareTo(startRow, endKey) < 0) &&
+ (scanWithNoUpperBound || Bytes.compareTo(stopRow, startKey) > 0)) {
+
+ final byte[] splitStart = scanWithNoLowerBound || Bytes.compareTo(startKey, startRow) >= 0 ? startKey : startRow;
+ final byte[] splitStop = (scanWithNoUpperBound || Bytes.compareTo(endKey, stopRow) <= 0)
+ && !isLastRegion ? endKey : stopRow;
+ int id = splits.size();
+ final TableInputSplit split = new TableInputSplit(id, hosts, table.getTableName(), splitStart, splitStop);
+ splits.add(split);
+ }
+ }
+ LOG.info("Created " + splits.size() + " splits");
+ for (TableInputSplit split : splits) {
+ logSplitInfo("created", split);
+ }
+ return splits.toArray(new TableInputSplit[0]);
+ }
+
+ private void logSplitInfo(String action, TableInputSplit split) {
+ int splitId = split.getSplitNumber();
+ String splitStart = Bytes.toString(split.getStartRow());
+ String splitEnd = Bytes.toString(split.getEndRow());
+ String splitStartKey = splitStart.isEmpty() ? "-" : splitStart;
+ String splitStopKey = splitEnd.isEmpty() ? "-" : splitEnd;
+ String[] hostnames = split.getHostnames();
+ LOG.info("{} split [{}|{}|{}|{}]",action, splitId, hostnames, splitStartKey, splitStopKey);
+ }
+
+ /**
+ * Test if the given region is to be included in the InputSplit while splitting the regions of a table.
+ * <p>
+ * This optimization is effective when there is a specific reasoning to exclude an entire region from the M-R job,
+ * (and hence, not contributing to the InputSplit), given the start and end keys of the same. <br>
+ * Useful when we need to remember the last-processed top record and revisit the [last, current) interval for M-R
+ * processing, continuously. In addition to reducing InputSplits, reduces the load on the region server as well, due
+ * to the ordering of the keys. <br>
+ * <br>
+ * Note: It is possible that <code>endKey.length() == 0 </code> , for the last (recent) region. <br>
+ * Override this method, if you want to bulk exclude regions altogether from M-R. By default, no region is excluded(
+ * i.e. all regions are included).
+ *
+ * @param startKey
+ * Start key of the region
+ * @param endKey
+ * End key of the region
+ * @return true, if this region needs to be included as part of the input (default).
+ */
+ private static boolean includeRegionInSplit(final byte[] startKey, final byte[] endKey) {
+ return true;
+ }
+
+ @Override
+ public InputSplitAssigner getInputSplitAssigner(TableInputSplit[] inputSplits) {
+ return new LocatableInputSplitAssigner(inputSplits);
+ }
+
+ @Override
+ public BaseStatistics getStatistics(BaseStatistics cachedStatistics) {
+ return null;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputSplit.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputSplit.java b/flink-staging/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputSplit.java
new file mode 100644
index 0000000..6d8bf42
--- /dev/null
+++ b/flink-staging/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputSplit.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.addons.hbase;
+
+import java.io.IOException;
+
+import org.apache.flink.core.io.LocatableInputSplit;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+/**
+ * This class implements a input splits for HBase. Each table input split corresponds to a key range (low, high). All
+ * references to row below refer to the key of the row.
+ */
+public class TableInputSplit extends LocatableInputSplit {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The name of the table to retrieve data from
+ */
+ private byte[] tableName;
+
+ /**
+ * The start row of the split.
+ */
+ private byte[] startRow;
+
+ /**
+ * The end row of the split.
+ */
+ private byte[] endRow;
+
+ /**
+ * Creates a new table input split
+ *
+ * @param splitNumber
+ * the number of the input split
+ * @param hostnames
+ * the names of the hosts storing the data the input split refers to
+ * @param tableName
+ * the name of the table to retrieve data from
+ * @param startRow
+ * the start row of the split
+ * @param endRow
+ * the end row of the split
+ */
+ TableInputSplit(final int splitNumber, final String[] hostnames, final byte[] tableName, final byte[] startRow,
+ final byte[] endRow) {
+ super(splitNumber, hostnames);
+
+ this.tableName = tableName;
+ this.startRow = startRow;
+ this.endRow = endRow;
+ }
+
+ /**
+ * Default constructor for serialization/deserialization.
+ */
+ public TableInputSplit() {
+ super();
+
+ this.tableName = null;
+ this.startRow = null;
+ this.endRow = null;
+ }
+
+ /**
+ * Returns the table name.
+ *
+ * @return The table name.
+ */
+ public byte[] getTableName() {
+ return this.tableName;
+ }
+
+ /**
+ * Returns the start row.
+ *
+ * @return The start row.
+ */
+ public byte[] getStartRow() {
+ return this.startRow;
+ }
+
+ /**
+ * Returns the end row.
+ *
+ * @return The end row.
+ */
+ public byte[] getEndRow() {
+ return this.endRow;
+ }
+
+
+ @Override
+ public void write(final DataOutputView out) throws IOException {
+
+ super.write(out);
+
+ // Write the table name
+ if (this.tableName == null) {
+ out.writeInt(-1);
+ } else {
+ out.writeInt(this.tableName.length);
+ out.write(this.tableName);
+ }
+
+ // Write the start row
+ if (this.startRow == null) {
+ out.writeInt(-1);
+ } else {
+ out.writeInt(this.startRow.length);
+ out.write(this.startRow);
+ }
+
+ // Write the end row
+ if (this.endRow == null) {
+ out.writeInt(-1);
+ } else {
+ out.writeInt(this.endRow.length);
+ out.write(this.endRow);
+ }
+ }
+
+
+ @Override
+ public void read(final DataInputView in) throws IOException {
+
+ super.read(in);
+
+ // Read the table name
+ int len = in.readInt();
+ if (len >= 0) {
+ this.tableName = new byte[len];
+ in.readFully(this.tableName);
+ }
+
+ // Read the start row
+ len = in.readInt();
+ if (len >= 0) {
+ this.startRow = new byte[len];
+ in.readFully(this.startRow);
+ }
+
+ // Read the end row
+ len = in.readInt();
+ if (len >= 0) {
+ this.endRow = new byte[len];
+ in.readFully(this.endRow);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java b/flink-staging/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java
new file mode 100755
index 0000000..b6f345a
--- /dev/null
+++ b/flink-staging/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.addons.hbase.example;
+
+import org.apache.flink.addons.hbase.TableInputFormat;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Simple stub for HBase DataSet
+ *
+ * To run the test first create the test table with hbase shell.
+ *
+ * Use the following commands:
+ * <ul>
+ * <li>create 'test-table', 'someCf'</li>
+ * <li>put 'test-table', '1', 'someCf:someQual', 'someString'</li>
+ * <li>put 'test-table', '2', 'someCf:someQual', 'anotherString'</li>
+ * </ul>
+ *
+ * The test should return just the first entry.
+ *
+ */
+public class HBaseReadExample {
+ public static void main(String[] args) throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ @SuppressWarnings("serial")
+ DataSet<Tuple2<String, String>> hbaseDs = env.createInput(new TableInputFormat<Tuple2<String, String>>() {
+ private final byte[] CF_SOME = "someCf".getBytes();
+ private final byte[] Q_SOME = "someQual".getBytes();
+ @Override
+ public String getTableName() {
+ return "test-table";
+ }
+
+ @Override
+ protected Scan getScanner() {
+ Scan scan = new Scan();
+ scan.addColumn(CF_SOME, Q_SOME);
+ return scan;
+ }
+
+ private Tuple2<String, String> reuse = new Tuple2<String, String>();
+
+ @Override
+ protected Tuple2<String, String> mapResultToTuple(Result r) {
+ String key = Bytes.toString(r.getRow());
+ String val = Bytes.toString(r.getValue(CF_SOME, Q_SOME));
+ reuse.setField(key, 0);
+ reuse.setField(val, 1);
+ return reuse;
+ }
+ })
+ .filter(new FilterFunction<Tuple2<String,String>>() {
+
+ @Override
+ public boolean filter(Tuple2<String, String> t) throws Exception {
+ String val = t.getField(1);
+ if(val.startsWith("someStr"))
+ return true;
+ return false;
+ }
+ });
+
+ hbaseDs.print();
+
+ // kick off execution.
+ env.execute();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hbase/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hbase/src/test/resources/hbase-site.xml b/flink-staging/flink-hbase/src/test/resources/hbase-site.xml
new file mode 100644
index 0000000..2984063
--- /dev/null
+++ b/flink-staging/flink-hbase/src/test/resources/hbase-site.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+ <property>
+ <name>hbase.tmp.dir</name>
+ <!--
+ <value>/media/Dati/hbase-0.98-data</value>
+ -->
+ <value>/opt/hbase-0.98.6.1-hadoop2/data</value>
+
+ </property>
+ <property>
+ <name>hbase.zookeeper.quorum</name>
+ <value>localhost</value>
+ </property>
+ <!--
+ <property>
+ <name>hadoop.security.group.mapping</name>
+ <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
+ </property>
+ -->
+</configuration>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hbase/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hbase/src/test/resources/log4j.properties b/flink-staging/flink-hbase/src/test/resources/log4j.properties
new file mode 100755
index 0000000..d6eb2b2
--- /dev/null
+++ b/flink-staging/flink-hbase/src/test/resources/log4j.properties
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=${hadoop.root.logger}
+hadoop.root.logger=INFO,console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/pom.xml b/flink-staging/flink-jdbc/pom.xml
new file mode 100644
index 0000000..7182984
--- /dev/null
+++ b/flink-staging/flink-jdbc/pom.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-staging</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-jdbc</artifactId>
+ <name>flink-jdbc</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>10.10.1.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
new file mode 100644
index 0000000..3cfaeb9
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.io.jdbc;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.apache.flink.api.common.io.NonParallelInput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.flink.api.common.io.DefaultInputSplitAssigner;
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.io.statistics.BaseStatistics;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.io.GenericInputSplit;
+import org.apache.flink.core.io.InputSplit;
+import org.apache.flink.core.io.InputSplitAssigner;
+import org.apache.flink.types.NullValue;
+
+/**
+ * InputFormat to read data from a database and generate tuples.
+ * The InputFormat has to be configured using the supplied InputFormatBuilder.
+ *
+ * @param <OUT>
+ * @see Tuple
+ * @see DriverManager
+ */
+public class JDBCInputFormat<OUT extends Tuple> implements InputFormat<OUT, InputSplit>, NonParallelInput {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(JDBCInputFormat.class);
+
+ private String username;
+ private String password;
+ private String drivername;
+ private String dbURL;
+ private String query;
+
+ private transient Connection dbConn;
+ private transient Statement statement;
+ private transient ResultSet resultSet;
+
+ private int[] columnTypes = null;
+
+ public JDBCInputFormat() {
+ }
+
+ @Override
+ public void configure(Configuration parameters) {
+ }
+
+ /**
+ * Connects to the source database and executes the query.
+ *
+ * @param ignored
+ * @throws IOException
+ */
+ @Override
+ public void open(InputSplit ignored) throws IOException {
+ try {
+ establishConnection();
+ statement = dbConn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
+ resultSet = statement.executeQuery(query);
+ } catch (SQLException se) {
+ close();
+ throw new IllegalArgumentException("open() failed." + se.getMessage(), se);
+ } catch (ClassNotFoundException cnfe) {
+ throw new IllegalArgumentException("JDBC-Class not found. - " + cnfe.getMessage(), cnfe);
+ }
+ }
+
+ private void establishConnection() throws SQLException, ClassNotFoundException {
+ Class.forName(drivername);
+ if (username == null) {
+ dbConn = DriverManager.getConnection(dbURL);
+ } else {
+ dbConn = DriverManager.getConnection(dbURL, username, password);
+ }
+ }
+
+ /**
+ * Closes all resources used.
+ *
+ * @throws IOException Indicates that a resource could not be closed.
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ resultSet.close();
+ } catch (SQLException se) {
+ LOG.info("Inputformat couldn't be closed - " + se.getMessage());
+ } catch (NullPointerException npe) {
+ }
+ try {
+ statement.close();
+ } catch (SQLException se) {
+ LOG.info("Inputformat couldn't be closed - " + se.getMessage());
+ } catch (NullPointerException npe) {
+ }
+ try {
+ dbConn.close();
+ } catch (SQLException se) {
+ LOG.info("Inputformat couldn't be closed - " + se.getMessage());
+ } catch (NullPointerException npe) {
+ }
+ }
+
+ /**
+ * Checks whether all data has been read.
+ *
+ * @return boolean value indication whether all data has been read.
+ * @throws IOException
+ */
+ @Override
+ public boolean reachedEnd() throws IOException {
+ try {
+ if (resultSet.isLast()) {
+ close();
+ return true;
+ }
+ return false;
+ } catch (SQLException se) {
+ throw new IOException("Couldn't evaluate reachedEnd() - " + se.getMessage(), se);
+ }
+ }
+
+ /**
+ * Stores the next resultSet row in a tuple
+ *
+ * @param tuple
+ * @return tuple containing next row
+ * @throws java.io.IOException
+ */
+ @Override
+ public OUT nextRecord(OUT tuple) throws IOException {
+ try {
+ resultSet.next();
+ if (columnTypes == null) {
+ extractTypes(tuple);
+ }
+ addValue(tuple);
+ return tuple;
+ } catch (SQLException se) {
+ close();
+ throw new IOException("Couldn't read data - " + se.getMessage(), se);
+ } catch (NullPointerException npe) {
+ close();
+ throw new IOException("Couldn't access resultSet", npe);
+ }
+ }
+
+ private void extractTypes(OUT tuple) throws SQLException, IOException {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ columnTypes = new int[resultSetMetaData.getColumnCount()];
+ if (tuple.getArity() != columnTypes.length) {
+ close();
+ throw new IOException("Tuple size does not match columncount");
+ }
+ for (int pos = 0; pos < columnTypes.length; pos++) {
+ columnTypes[pos] = resultSetMetaData.getColumnType(pos + 1);
+ }
+ }
+
+ /**
+ * Enters data value from the current resultSet into a Record.
+ *
+ * @param reuse Target Record.
+ */
+ private void addValue(OUT reuse) throws SQLException {
+ for (int pos = 0; pos < columnTypes.length; pos++) {
+ switch (columnTypes[pos]) {
+ case java.sql.Types.NULL:
+ reuse.setField(NullValue.getInstance(), pos);
+ break;
+ case java.sql.Types.BOOLEAN:
+ reuse.setField(resultSet.getBoolean(pos + 1), pos);
+ break;
+ case java.sql.Types.BIT:
+ reuse.setField(resultSet.getBoolean(pos + 1), pos);
+ break;
+ case java.sql.Types.CHAR:
+ reuse.setField(resultSet.getString(pos + 1), pos);
+ break;
+ case java.sql.Types.NCHAR:
+ reuse.setField(resultSet.getString(pos + 1), pos);
+ break;
+ case java.sql.Types.VARCHAR:
+ reuse.setField(resultSet.getString(pos + 1), pos);
+ break;
+ case java.sql.Types.LONGVARCHAR:
+ reuse.setField(resultSet.getString(pos + 1), pos);
+ break;
+ case java.sql.Types.LONGNVARCHAR:
+ reuse.setField(resultSet.getString(pos + 1), pos);
+ break;
+ case java.sql.Types.TINYINT:
+ reuse.setField(resultSet.getShort(pos + 1), pos);
+ break;
+ case java.sql.Types.SMALLINT:
+ reuse.setField(resultSet.getShort(pos + 1), pos);
+ break;
+ case java.sql.Types.BIGINT:
+ reuse.setField(resultSet.getLong(pos + 1), pos);
+ break;
+ case java.sql.Types.INTEGER:
+ reuse.setField(resultSet.getInt(pos + 1), pos);
+ break;
+ case java.sql.Types.FLOAT:
+ reuse.setField(resultSet.getDouble(pos + 1), pos);
+ break;
+ case java.sql.Types.REAL:
+ reuse.setField(resultSet.getFloat(pos + 1), pos);
+ break;
+ case java.sql.Types.DOUBLE:
+ reuse.setField(resultSet.getDouble(pos + 1), pos);
+ break;
+ case java.sql.Types.DECIMAL:
+ reuse.setField(resultSet.getBigDecimal(pos + 1).doubleValue(), pos);
+ break;
+ case java.sql.Types.NUMERIC:
+ reuse.setField(resultSet.getBigDecimal(pos + 1).doubleValue(), pos);
+ break;
+ case java.sql.Types.DATE:
+ reuse.setField(resultSet.getDate(pos + 1).toString(), pos);
+ break;
+ case java.sql.Types.TIME:
+ reuse.setField(resultSet.getTime(pos + 1).getTime(), pos);
+ break;
+ case java.sql.Types.TIMESTAMP:
+ reuse.setField(resultSet.getTimestamp(pos + 1).toString(), pos);
+ break;
+ case java.sql.Types.SQLXML:
+ reuse.setField(resultSet.getSQLXML(pos + 1).toString(), pos);
+ break;
+ default:
+ throw new SQLException("Unsupported sql-type [" + columnTypes[pos] + "] on column [" + pos + "]");
+
+ // case java.sql.Types.BINARY:
+ // case java.sql.Types.VARBINARY:
+ // case java.sql.Types.LONGVARBINARY:
+ // case java.sql.Types.ARRAY:
+ // case java.sql.Types.JAVA_OBJECT:
+ // case java.sql.Types.BLOB:
+ // case java.sql.Types.CLOB:
+ // case java.sql.Types.NCLOB:
+ // case java.sql.Types.DATALINK:
+ // case java.sql.Types.DISTINCT:
+ // case java.sql.Types.OTHER:
+ // case java.sql.Types.REF:
+ // case java.sql.Types.ROWID:
+ // case java.sql.Types.STRUCT:
+ }
+ }
+ }
+
+ @Override
+ public BaseStatistics getStatistics(BaseStatistics cachedStatistics) throws IOException {
+ return cachedStatistics;
+ }
+
+ @Override
+ public InputSplit[] createInputSplits(int minNumSplits) throws IOException {
+ GenericInputSplit[] split = {
+ new GenericInputSplit(0, 1)
+ };
+ return split;
+ }
+
+ @Override
+ public InputSplitAssigner getInputSplitAssigner(InputSplit[] inputSplits) {
+ return new DefaultInputSplitAssigner(inputSplits);
+ }
+
+
+ /**
+ * A builder used to set parameters to the output format's configuration in a fluent way.
+ * @return builder
+ */
+ public static JDBCInputFormatBuilder buildJDBCInputFormat() {
+ return new JDBCInputFormatBuilder();
+ }
+
+ public static class JDBCInputFormatBuilder {
+ private final JDBCInputFormat format;
+
+ public JDBCInputFormatBuilder() {
+ this.format = new JDBCInputFormat();
+ }
+
+ public JDBCInputFormatBuilder setUsername(String username) {
+ format.username = username;
+ return this;
+ }
+
+ public JDBCInputFormatBuilder setPassword(String password) {
+ format.password = password;
+ return this;
+ }
+
+ public JDBCInputFormatBuilder setDrivername(String drivername) {
+ format.drivername = drivername;
+ return this;
+ }
+
+ public JDBCInputFormatBuilder setDBUrl(String dbURL) {
+ format.dbURL = dbURL;
+ return this;
+ }
+
+ public JDBCInputFormatBuilder setQuery(String query) {
+ format.query = query;
+ return this;
+ }
+
+ public JDBCInputFormat finish() {
+ if (format.username == null) {
+ LOG.info("Username was not supplied separately.");
+ }
+ if (format.password == null) {
+ LOG.info("Password was not supplied separately.");
+ }
+ if (format.dbURL == null) {
+ throw new IllegalArgumentException("No dababase URL supplied.");
+ }
+ if (format.query == null) {
+ throw new IllegalArgumentException("No query suplied");
+ }
+ if (format.drivername == null) {
+ throw new IllegalArgumentException("No driver supplied");
+ }
+ return format;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
new file mode 100644
index 0000000..6771772
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.io.jdbc;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.flink.api.common.io.OutputFormat;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.configuration.Configuration;
+
+/**
+ * OutputFormat to write tuples into a database.
+ * The OutputFormat has to be configured using the supplied OutputFormatBuilder.
+ *
+ * @param <OUT>
+ * @see Tuple
+ * @see DriverManager
+ */
+public class JDBCOutputFormat<OUT extends Tuple> implements OutputFormat<OUT> {
+ private static final long serialVersionUID = 1L;
+
+ @SuppressWarnings("unused")
+ private static final Logger LOG = LoggerFactory.getLogger(JDBCOutputFormat.class);
+
+ private String username;
+ private String password;
+ private String drivername;
+ private String dbURL;
+ private String query;
+ private int batchInterval = 5000;
+
+ private Connection dbConn;
+ private PreparedStatement upload;
+
+ private SupportedTypes[] types = null;
+
+ private int batchCount = 0;
+
+ public JDBCOutputFormat() {
+ }
+
+ @Override
+ public void configure(Configuration parameters) {
+ }
+
+ /**
+ * Connects to the target database and initializes the prepared statement.
+ *
+ * @param taskNumber The number of the parallel instance.
+ * @throws IOException Thrown, if the output could not be opened due to an
+ * I/O problem.
+ */
+ @Override
+ public void open(int taskNumber, int numTasks) throws IOException {
+ try {
+ establishConnection();
+ upload = dbConn.prepareStatement(query);
+ } catch (SQLException sqe) {
+ close();
+ throw new IllegalArgumentException("open() failed:\t!", sqe);
+ } catch (ClassNotFoundException cnfe) {
+ close();
+ throw new IllegalArgumentException("JDBC-Class not found:\t", cnfe);
+ }
+ }
+
+ private void establishConnection() throws SQLException, ClassNotFoundException {
+ Class.forName(drivername);
+ if (username == null) {
+ dbConn = DriverManager.getConnection(dbURL);
+ } else {
+ dbConn = DriverManager.getConnection(dbURL, username, password);
+ }
+ }
+
+ private enum SupportedTypes {
+ BOOLEAN,
+ BYTE,
+ SHORT,
+ INTEGER,
+ LONG,
+ STRING,
+ FLOAT,
+ DOUBLE
+ }
+
+ /**
+ * Adds a record to the prepared statement.
+ * <p>
+ * When this method is called, the output format is guaranteed to be opened.
+ *
+ * @param tuple The records to add to the output.
+ * @throws IOException Thrown, if the records could not be added due to an I/O problem.
+ */
+ @Override
+ public void writeRecord(OUT tuple) throws IOException {
+ try {
+ if (types == null) {
+ extractTypes(tuple);
+ }
+ addValues(tuple);
+ upload.addBatch();
+ batchCount++;
+ if (batchCount >= batchInterval) {
+ upload.executeBatch();
+ batchCount = 0;
+ }
+ } catch (SQLException sqe) {
+ close();
+ throw new IllegalArgumentException("writeRecord() failed", sqe);
+ } catch (IllegalArgumentException iae) {
+ close();
+ throw new IllegalArgumentException("writeRecord() failed", iae);
+ }
+ }
+
+ private void extractTypes(OUT tuple) {
+ types = new SupportedTypes[tuple.getArity()];
+ for (int x = 0; x < tuple.getArity(); x++) {
+ types[x] = SupportedTypes.valueOf(tuple.getField(x).getClass().getSimpleName().toUpperCase());
+ }
+ }
+
+ private void addValues(OUT tuple) throws SQLException {
+ for (int index = 0; index < tuple.getArity(); index++) {
+ switch (types[index]) {
+ case BOOLEAN:
+ upload.setBoolean(index + 1, (Boolean) tuple.getField(index));
+ break;
+ case BYTE:
+ upload.setByte(index + 1, (Byte) tuple.getField(index));
+ break;
+ case SHORT:
+ upload.setShort(index + 1, (Short) tuple.getField(index));
+ break;
+ case INTEGER:
+ upload.setInt(index + 1, (Integer) tuple.getField(index));
+ break;
+ case LONG:
+ upload.setLong(index + 1, (Long) tuple.getField(index));
+ break;
+ case STRING:
+ upload.setString(index + 1, (String) tuple.getField(index));
+ break;
+ case FLOAT:
+ upload.setFloat(index + 1, (Float) tuple.getField(index));
+ break;
+ case DOUBLE:
+ upload.setDouble(index + 1, (Double) tuple.getField(index));
+ break;
+ }
+ }
+ }
+
+ /**
+ * Executes prepared statement and closes all resources of this instance.
+ *
+ * @throws IOException Thrown, if the input could not be closed properly.
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ upload.executeBatch();
+ batchCount = 0;
+ } catch (SQLException se) {
+ throw new IllegalArgumentException("close() failed", se);
+ } catch (NullPointerException se) {
+ }
+ try {
+ upload.close();
+ } catch (SQLException se) {
+ LOG.info("Inputformat couldn't be closed - " + se.getMessage());
+ } catch (NullPointerException npe) {
+ }
+ try {
+ dbConn.close();
+ } catch (SQLException se) {
+ LOG.info("Inputformat couldn't be closed - " + se.getMessage());
+ } catch (NullPointerException npe) {
+ }
+ }
+
+ public static JDBCOutputFormatBuilder buildJDBCOutputFormat() {
+ return new JDBCOutputFormatBuilder();
+ }
+
+ public static class JDBCOutputFormatBuilder {
+ private final JDBCOutputFormat format;
+
+ protected JDBCOutputFormatBuilder() {
+ this.format = new JDBCOutputFormat();
+ }
+
+ public JDBCOutputFormatBuilder setUsername(String username) {
+ format.username = username;
+ return this;
+ }
+
+ public JDBCOutputFormatBuilder setPassword(String password) {
+ format.password = password;
+ return this;
+ }
+
+ public JDBCOutputFormatBuilder setDrivername(String drivername) {
+ format.drivername = drivername;
+ return this;
+ }
+
+ public JDBCOutputFormatBuilder setDBUrl(String dbURL) {
+ format.dbURL = dbURL;
+ return this;
+ }
+
+ public JDBCOutputFormatBuilder setQuery(String query) {
+ format.query = query;
+ return this;
+ }
+
+ public JDBCOutputFormatBuilder setBatchInterval(int batchInterval) {
+ format.batchInterval = batchInterval;
+ return this;
+ }
+
+ /**
+ Finalizes the configuration and checks validity.
+ @return Configured JDBCOutputFormat
+ */
+ public JDBCOutputFormat finish() {
+ if (format.username == null) {
+ LOG.info("Username was not supplied separately.");
+ }
+ if (format.password == null) {
+ LOG.info("Password was not supplied separately.");
+ }
+ if (format.dbURL == null) {
+ throw new IllegalArgumentException("No dababase URL supplied.");
+ }
+ if (format.query == null) {
+ throw new IllegalArgumentException("No query suplied");
+ }
+ if (format.drivername == null) {
+ throw new IllegalArgumentException("No driver supplied");
+ }
+ return format;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/example/JDBCExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/example/JDBCExample.java b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/example/JDBCExample.java
new file mode 100644
index 0000000..7b012ba
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/example/JDBCExample.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.io.jdbc.example;
+
+import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.DOUBLE_TYPE_INFO;
+import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.INT_TYPE_INFO;
+import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.jdbc.JDBCInputFormat;
+import org.apache.flink.api.java.io.jdbc.JDBCOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+
+public class JDBCExample {
+
+ public static void main(String[] args) throws Exception {
+ prepareTestDb();
+
+ ExecutionEnvironment environment = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple5> source
+ = environment.createInput(JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setDBUrl("jdbc:derby:memory:ebookshop")
+ .setQuery("select * from books")
+ .finish(),
+ new TupleTypeInfo(Tuple5.class, INT_TYPE_INFO, STRING_TYPE_INFO, STRING_TYPE_INFO, DOUBLE_TYPE_INFO, INT_TYPE_INFO)
+ );
+
+ source.output(JDBCOutputFormat.buildJDBCOutputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setDBUrl("jdbc:derby:memory:ebookshop")
+ .setQuery("insert into newbooks (id,title,author,price,qty) values (?,?,?,?,?)")
+ .finish());
+ environment.execute();
+ }
+
+ private static void prepareTestDb() throws Exception {
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ Connection conn = DriverManager.getConnection(dbURL);
+
+ StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE books (");
+ sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
+ sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
+ sqlQueryBuilder.append("qty INT DEFAULT NULL,");
+ sqlQueryBuilder.append("PRIMARY KEY (id))");
+
+ Statement stat = conn.createStatement();
+ stat.executeUpdate(sqlQueryBuilder.toString());
+ stat.close();
+
+ sqlQueryBuilder = new StringBuilder("CREATE TABLE newbooks (");
+ sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
+ sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
+ sqlQueryBuilder.append("qty INT DEFAULT NULL,");
+ sqlQueryBuilder.append("PRIMARY KEY (id))");
+
+ stat = conn.createStatement();
+ stat.executeUpdate(sqlQueryBuilder.toString());
+ stat.close();
+
+ sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
+ sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
+ sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
+ sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
+ sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
+ sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
+
+ stat = conn.createStatement();
+ stat.execute(sqlQueryBuilder.toString());
+ stat.close();
+
+ conn.close();
+ }
+}
[05/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/pom.xml b/flink-staging/flink-streaming/flink-streaming-examples/pom.xml
new file mode 100644
index 0000000..3661726
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/pom.xml
@@ -0,0 +1,518 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-streaming-examples</artifactId>
+ <name>flink-streaming-examples</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-scala</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java-examples</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-connectors</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- get default data from flink-java-examples package -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.9</version><!--$NO-MVN-MAN-VER$-->
+ <executions>
+ <execution>
+ <id>unpack</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java-examples</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ <overWrite>false</overWrite>
+ <outputDirectory>${project.build.directory}/classes</outputDirectory>
+ <includes>org/apache/flink/examples/java/wordcount/util/WordCountData.class</includes>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- self-contained jars for each example -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+
+ <executions>
+
+ <!-- Iteration -->
+ <execution>
+ <id>Iteration</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>Iteration</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.iteration.IterateExample</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/iteration/*.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
+ <!-- IncrementalLearning -->
+ <execution>
+ <id>IncrementalLearning</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>IncrementalLearning</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.ml.IncrementalLearningSkeleton</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/ml/*.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
+ <!-- Twitter -->
+ <execution>
+ <id>Twitter</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>Twitter</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.twitter.TwitterStream</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/twitter/*.class</include>
+ <include>org/apache/flink/streaming/examples/twitter/util/*.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
+ <!-- WindowJoin -->
+ <execution>
+ <id>WindowJoin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>WindowJoin</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.join.WindowJoin</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/join/*.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
+ <!-- WordCountPOJO -->
+ <execution>
+ <id>WordCountPOJO</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>WordCountPOJO</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.wordcount.PojoExample</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/wordcount/PojoExample.class</include>
+ <include>org/apache/flink/streaming/examples/wordcount/PojoExample$*.class</include>
+ <include>org/apache/flink/examples/java/wordcount/util/WordCountData.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
+ <!-- WordCount -->
+ <execution>
+ <id>WordCount</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>WordCount</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.wordcount.WordCount</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/wordcount/WordCount.class</include>
+ <include>org/apache/flink/streaming/examples/wordcount/WordCount$*.class</include>
+ <include>org/apache/flink/examples/java/wordcount/util/WordCountData.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
+ <!-- SocketTextStreamWordCount -->
+ <execution>
+ <id>SocketTextStreamWordCount</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>SocketTextStreamWordCount</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.socket.SocketTextStreamWordCount</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.class</include>
+ <include>org/apache/flink/streaming/examples/wordcount/WordCount$Tokenizer.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
+ <!-- DeltaExract -->
+ <execution>
+ <id>DeltaExract</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>DeltaExract</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.windowing.DeltaExtractExample</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/windowing/DeltaExtractExample.class</include>
+ <include>org/apache/flink/streaming/examples/windowing/DeltaExtractExample$*.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
+ <!-- MultiplePolicies -->
+ <execution>
+ <id>MultiplePolicies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>MultiplePolicies</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.windowing.MultiplePoliciesExample</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.class</include>
+ <include>org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample$*.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
+ <!-- SlidingExample -->
+ <execution>
+ <id>SlidingExample</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>SlidingExample</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.windowing.SlidingExample</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/windowing/SlidingExample.class</include>
+ <include>org/apache/flink/streaming/examples/windowing/SlidingExample$*.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
+ <!-- TimeWindowing -->
+ <execution>
+ <id>TimeWindowing</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>TimeWindowing</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.streaming.examples.windowing.TimeWindowingExample</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>org/apache/flink/streaming/examples/windowing/TimeWindowingExample.class</include>
+ <include>org/apache/flink/streaming/examples/windowing/TimeWindowingExample$*.class</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+<!-- Scala Compiler -->
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.1.4</version>
+ <executions>
+ <!-- Run scala compiler in the process-resources phase, so that dependencies on
+ scala classes can be resolved later in the (Java) compile phase -->
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+
+ <!-- Run scala compiler in the process-test-resources phase, so that dependencies on
+ scala classes can be resolved later in the (Java) test-compile phase -->
+ <execution>
+ <id>scala-test-compile</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <jvmArgs>
+ <jvmArg>-Xms128m</jvmArg>
+ <jvmArg>-Xmx512m</jvmArg>
+ </jvmArgs>
+ <compilerPlugins>
+ <compilerPlugin>
+ <groupId>org.scalamacros</groupId>
+ <artifactId>paradise_${scala.version}</artifactId>
+ <version>${scala.macros.version}</version>
+ </compilerPlugin>
+ </compilerPlugins>
+ </configuration>
+ </plugin>
+
+ <!-- Eclipse Integration -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.8</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ <projectnatures>
+ <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
+ <projectnature>org.eclipse.jdt.core.javanature</projectnature>
+ </projectnatures>
+ <buildcommands>
+ <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
+ </buildcommands>
+ <classpathContainers>
+ <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
+ <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
+ </classpathContainers>
+ <excludes>
+ <exclude>org.scala-lang:scala-library</exclude>
+ <exclude>org.scala-lang:scala-compiler</exclude>
+ </excludes>
+ <sourceIncludes>
+ <sourceInclude>**/*.scala</sourceInclude>
+ <sourceInclude>**/*.java</sourceInclude>
+ </sourceIncludes>
+ </configuration>
+ </plugin>
+
+ <!-- Adding scala source directories to build path -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <!-- Add src/main/scala to eclipse build path -->
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ <!-- Add src/test/scala to eclipse build path -->
+ <execution>
+ <id>add-test-source</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/test/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
+ <version>0.5.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <verbose>false</verbose>
+ <failOnViolation>true</failOnViolation>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <failOnWarning>false</failOnWarning>
+ <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
+ <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
+ <configLocation>${project.basedir}/../../../tools/maven/scalastyle-config.xml</configLocation>
+ <outputFile>${project.basedir}/scalastyle-output.xml</outputFile>
+ <outputEncoding>UTF-8</outputEncoding>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <versionRange>[2.9,)</versionRange>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore/>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
new file mode 100644
index 0000000..998e818
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.iteration;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.IterativeDataStream;
+import org.apache.flink.streaming.api.datastream.SplitDataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
+/**
+ * Example illustrating iterations in Flink streaming.
+ *
+ * <p>
+ * The program sums up random numbers and counts additions it performs to reach
+ * a specific threshold in an iterative streaming fashion.
+ * </p>
+ *
+ * <p>
+ * This example shows how to use:
+ * <ul>
+ * <li>streaming iterations,
+ * <li>buffer timeout to enhance latency,
+ * <li>directed outputs.
+ * </ul>
+ */
+public class IterateExample {
+
+ // *************************************************************************
+ // PROGRAM
+ // *************************************************************************
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ // set up input for the stream of (0,0) pairs
+ List<Tuple2<Double, Integer>> input = new ArrayList<Tuple2<Double, Integer>>();
+ for (int i = 0; i < 1000; i++) {
+ input.add(new Tuple2<Double, Integer>(0., 0));
+ }
+
+ // obtain execution environment and set setBufferTimeout(0) to enable
+ // continuous flushing of the output buffers (lowest latency)
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment()
+ .setBufferTimeout(1);
+
+ // create an iterative data stream from the input with 5 second timeout
+ IterativeDataStream<Tuple2<Double, Integer>> it = env.fromCollection(input).shuffle()
+ .iterate(5000);
+
+ // apply the step function to add new random value to the tuple and to
+ // increment the counter and split the output with the output selector
+ SplitDataStream<Tuple2<Double, Integer>> step = it.map(new Step()).split(new MySelector());
+
+ // close the iteration by selecting the tuples that were directed to the
+ // 'iterate' channel in the output selector
+ it.closeWith(step.select("iterate"));
+
+ // to produce the final output select the tuples directed to the
+ // 'output' channel then project it to the desired second field
+
+ DataStream<Tuple1<Integer>> numbers = step.select("output").project(1).types(Integer.class);
+
+ // emit result
+ if (fileOutput) {
+ numbers.writeAsText(outputPath, 1);
+ } else {
+ numbers.print();
+ }
+
+ // execute the program
+ env.execute("Streaming Iteration Example");
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ /**
+ * Iteration step function which takes an input (Double , Integer) and
+ * produces an output (Double + random, Integer + 1).
+ */
+ public static class Step extends
+ RichMapFunction<Tuple2<Double, Integer>, Tuple2<Double, Integer>> {
+ private static final long serialVersionUID = 1L;
+ private transient Random rnd;
+
+ public void open(Configuration parameters) {
+ rnd = new Random();
+ }
+
+ @Override
+ public Tuple2<Double, Integer> map(Tuple2<Double, Integer> value) throws Exception {
+ return new Tuple2<Double, Integer>(value.f0 + rnd.nextDouble(), value.f1 + 1);
+ }
+ }
+
+ /**
+ * OutputSelector testing which tuple needs to be iterated again.
+ */
+ public static class MySelector implements OutputSelector<Tuple2<Double, Integer>> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Iterable<String> select(Tuple2<Double, Integer> value) {
+ List<String> output = new ArrayList<String>();
+ if (value.f0 > 100) {
+ output.add("output");
+ } else {
+ output.add("iterate");
+ }
+ return output;
+ }
+
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if (args.length == 1) {
+ outputPath = args[0];
+ } else {
+ System.err.println("Usage: IterateExample <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing IterateExample with generated data.");
+ System.out.println(" Provide parameter to write to file.");
+ System.out.println(" Usage: IterateExample <result path>");
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
new file mode 100644
index 0000000..dcfed50
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.join;
+
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.RichSourceFunction;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.util.Collector;
+
+/**
+ * Example illustrating join over sliding windows of streams in Flink.
+ *
+ * <p>
+ * his example will join two streams with a sliding window. One which emits
+ * grades and one which emits salaries of people.
+ * </p>
+ *
+ * <p>
+ * This example shows how to:
+ * <ul>
+ * <li>do windowed joins,
+ * <li>use tuple data types,
+ * <li>write a simple streaming program.
+ */
+public class WindowJoin {
+
+ // *************************************************************************
+ // PROGRAM
+ // *************************************************************************
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ // obtain execution environment
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ // connect to the data sources for grades and salaries
+ DataStream<Tuple2<String, Integer>> grades = env.addSource(new GradeSource());
+ DataStream<Tuple2<String, Integer>> salaries = env.addSource(new SalarySource());
+
+ // apply a temporal join over the two stream based on the names over one
+ // second windows
+ DataStream<Tuple3<String, Integer, Integer>> joinedStream = grades
+ .join(salaries)
+ .onWindow(1, TimeUnit.SECONDS)
+ .where(0)
+ .equalTo(0)
+ .with(new MyJoinFunction());
+
+ // emit result
+ if (fileOutput) {
+ joinedStream.writeAsText(outputPath, 1);
+ } else {
+ joinedStream.print();
+ }
+
+ // execute program
+ env.execute("Windowed Join Example");
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ private final static String[] names = { "tom", "jerry", "alice", "bob", "john", "grace" };
+ private final static int GRADE_COUNT = 5;
+ private final static int SALARY_MAX = 10000;
+ private final static int SLEEP_TIME = 10;
+
+ /**
+ * Continuously emit tuples with random names and integers (grades).
+ */
+ public static class GradeSource implements SourceFunction<Tuple2<String, Integer>> {
+ private static final long serialVersionUID = 1L;
+
+ private Random rand;
+ private Tuple2<String, Integer> outTuple;
+
+ public GradeSource() {
+ rand = new Random();
+ outTuple = new Tuple2<String, Integer>();
+ }
+
+ @Override
+ public void invoke(Collector<Tuple2<String, Integer>> out) throws Exception {
+ while (true) {
+ outTuple.f0 = names[rand.nextInt(names.length)];
+ outTuple.f1 = rand.nextInt(GRADE_COUNT) + 1;
+ out.collect(outTuple);
+ Thread.sleep(rand.nextInt(SLEEP_TIME) + 1);
+ }
+ }
+ }
+
+ /**
+ * Continuously emit tuples with random names and integers (salaries).
+ */
+ public static class SalarySource extends RichSourceFunction<Tuple2<String, Integer>> {
+ private static final long serialVersionUID = 1L;
+
+ private transient Random rand;
+ private transient Tuple2<String, Integer> outTuple;
+
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ rand = new Random();
+ outTuple = new Tuple2<String, Integer>();
+ }
+
+ @Override
+ public void invoke(Collector<Tuple2<String, Integer>> out) throws Exception {
+ while (true) {
+ outTuple.f0 = names[rand.nextInt(names.length)];
+ outTuple.f1 = rand.nextInt(SALARY_MAX) + 1;
+ out.collect(outTuple);
+ Thread.sleep(rand.nextInt(SLEEP_TIME) + 1);
+ }
+ }
+ }
+
+ public static class MyJoinFunction
+ implements
+ JoinFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Integer, Integer>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private Tuple3<String, Integer, Integer> joined = new Tuple3<String, Integer, Integer>();
+
+ @Override
+ public Tuple3<String, Integer, Integer> join(Tuple2<String, Integer> first,
+ Tuple2<String, Integer> second) throws Exception {
+ joined.f0 = first.f0;
+ joined.f1 = first.f1;
+ joined.f2 = second.f1;
+ return joined;
+ }
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if (args.length == 1) {
+ outputPath = args[0];
+ } else {
+ System.err.println("Usage: WindowJoin <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing WindowJoin with generated data.");
+ System.out.println(" Provide parameter to write to file.");
+ System.out.println(" Usage: WindowJoin <result path>");
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
new file mode 100755
index 0000000..375c86d
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.ml;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.co.CoMapFunction;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.helper.Time;
+import org.apache.flink.util.Collector;
+
+/**
+ * Skeleton for incremental machine learning algorithm consisting of a
+ * pre-computed model, which gets updated for the new inputs and new input data
+ * for which the job provides predictions.
+ *
+ * <p>
+ * This may serve as a base of a number of algorithms, e.g. updating an
+ * incremental Alternating Least Squares model while also providing the
+ * predictions.
+ * </p>
+ *
+ * <p>
+ * This example shows how to use:
+ * <ul>
+ * <li>Connected streams
+ * <li>CoFunctions
+ * <li>Tuple data types
+ * </ul>
+ */
+public class IncrementalLearningSkeleton {
+
+ // *************************************************************************
+ // PROGRAM
+ // *************************************************************************
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ // build new model on every second of new data
+ DataStream<Double[]> model = env.addSource(new TrainingDataSource())
+ .window(Time.of(5000, TimeUnit.MILLISECONDS))
+ .reduceGroup(new PartialModelBuilder());
+
+ // use partial model for prediction
+ DataStream<Integer> prediction = env.addSource(new NewDataSource()).connect(model)
+ .map(new Predictor());
+
+ // emit result
+ if (fileOutput) {
+ prediction.writeAsText(outputPath, 1);
+ } else {
+ prediction.print();
+ }
+
+ // execute program
+ env.execute("Streaming Incremental Learning");
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ /**
+ * Feeds new data for prediction. By default it is implemented as constantly
+ * emitting the Integer 1 in a loop.
+ */
+ public static class NewDataSource implements SourceFunction<Integer> {
+ private static final long serialVersionUID = 1L;
+ private static final int NEW_DATA_SLEEP_TIME = 1000;
+
+ @Override
+ public void invoke(Collector<Integer> collector) throws Exception {
+ while (true) {
+ collector.collect(getNewData());
+ }
+ }
+
+ private Integer getNewData() throws InterruptedException {
+ Thread.sleep(NEW_DATA_SLEEP_TIME);
+ return 1;
+ }
+ }
+
+ /**
+ * Feeds new training data for the partial model builder. By default it is
+ * implemented as constantly emitting the Integer 1 in a loop.
+ */
+ public static class TrainingDataSource implements SourceFunction<Integer> {
+ private static final long serialVersionUID = 1L;
+ private static final int TRAINING_DATA_SLEEP_TIME = 10;
+
+ @Override
+ public void invoke(Collector<Integer> collector) throws Exception {
+ while (true) {
+ collector.collect(getTrainingData());
+ }
+
+ }
+
+ private Integer getTrainingData() throws InterruptedException {
+ Thread.sleep(TRAINING_DATA_SLEEP_TIME);
+ return 1;
+
+ }
+ }
+
+ /**
+ * Builds up-to-date partial models on new training data.
+ */
+ public static class PartialModelBuilder implements GroupReduceFunction<Integer, Double[]> {
+ private static final long serialVersionUID = 1L;
+
+ protected Double[] buildPartialModel(Iterable<Integer> values) {
+ return new Double[] { 1. };
+ }
+
+ @Override
+ public void reduce(Iterable<Integer> values, Collector<Double[]> out) throws Exception {
+ out.collect(buildPartialModel(values));
+ }
+ }
+
+ /**
+ * Creates prediction using the model produced in batch-processing and the
+ * up-to-date partial model.
+ *
+ * <p>
+ * By defaults emits the Integer 0 for every prediction and the Integer 1
+ * for every model update.
+ * </p>
+ */
+ public static class Predictor implements CoMapFunction<Integer, Double[], Integer> {
+ private static final long serialVersionUID = 1L;
+
+ Double[] batchModel = null;
+ Double[] partialModel = null;
+
+ @Override
+ public Integer map1(Integer value) {
+ // Return prediction
+ return predict(value);
+ }
+
+ @Override
+ public Integer map2(Double[] value) {
+ // Update model
+ partialModel = value;
+ batchModel = getBatchModel();
+ return 1;
+ }
+
+ // pulls model built with batch-job on the old training data
+ protected Double[] getBatchModel() {
+ return new Double[] { 0. };
+ }
+
+ // performs prediction using the two models
+ protected Integer predict(Integer inTuple) {
+ return 0;
+ }
+
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if (args.length == 1) {
+ outputPath = args[0];
+ } else {
+ System.err.println("Usage: IncrementalLearningSkeleton <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing IncrementalLearningSkeleton with generated data.");
+ System.out.println(" Provide parameter to write to file.");
+ System.out.println(" Usage: IncrementalLearningSkeleton <result path>");
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
new file mode 100644
index 0000000..e9b60f4
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.socket;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.examples.wordcount.WordCount.Tokenizer;
+
+/**
+ * This example shows an implementation of WordCount with data from a text
+ * socket. To run the example make sure that the service providing the text data
+ * is already up and running.
+ *
+ * <p>
+ * To start an example socket text stream on your local machine run netcat from
+ * a command line: <code>nc -lk 9999</code>, where the parameter specifies the
+ * port number.
+ *
+ *
+ * <p>
+ * Usage:
+ * <code>SocketTextStreamWordCount <hostname> <port> <result path></code>
+ * <br>
+ *
+ * <p>
+ * This example shows how to:
+ * <ul>
+ * <li>use StreamExecutionEnvironment.socketTextStream
+ * <li>write a simple Flink program,
+ * <li>write and use user-defined functions.
+ * </ul>
+ *
+ * @see <a href="www.openbsd.org/cgi-bin/man.cgi?query=nc">netcat</a>
+ */
+public class SocketTextStreamWordCount {
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ // set up the execution environment
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment
+ .getExecutionEnvironment();
+
+ // get input data
+ DataStream<String> text = env.socketTextStream(hostName, port);
+
+ DataStream<Tuple2<String, Integer>> counts =
+ // split up the lines in pairs (2-tuples) containing: (word,1)
+ text.flatMap(new Tokenizer())
+ // group by the tuple field "0" and sum up tuple field "1"
+ .groupBy(0)
+ .sum(1);
+
+ if (fileOutput) {
+ counts.writeAsText(outputPath, 1);
+ } else {
+ counts.print();
+ }
+
+ // execute program
+ env.execute("WordCount from SocketTextStream Example");
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String hostName;
+ private static int port;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ // parse input arguments
+ if (args.length == 3) {
+ fileOutput = true;
+ hostName = args[0];
+ port = Integer.valueOf(args[1]);
+ outputPath = args[2];
+ } else if (args.length == 2) {
+ hostName = args[0];
+ port = Integer.valueOf(args[1]);
+ } else {
+ System.err.println("Usage: SocketTextStreamWordCount <hostname> <port> [<output path>]");
+ return false;
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java
new file mode 100644
index 0000000..1901475
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.twitter;
+
+import java.util.StringTokenizer;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
+import org.apache.flink.streaming.examples.twitter.util.TwitterStreamData;
+import org.apache.flink.util.Collector;
+import org.apache.sling.commons.json.JSONException;
+
+/**
+ * Implements the "TwitterStream" program that computes a most used word
+ * occurrence over JSON files in a streaming fashion.
+ *
+ * <p>
+ * The input is a JSON text file with lines separated by newline characters.
+ *
+ * <p>
+ * Usage: <code>TwitterStream <text path></code><br>
+ * If no parameters are provided, the program is run with default data from
+ * {@link TwitterStreamData}.
+ *
+ * <p>
+ * This example shows how to:
+ * <ul>
+ * <li>acquire external data,
+ * <li>use in-line defined functions,
+ * <li>handle flattened stream inputs.
+ * </ul>
+ *
+ */
+public class TwitterStream {
+
+ // *************************************************************************
+ // PROGRAM
+ // *************************************************************************
+
+ public static void main(String[] args) throws Exception {
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ // set up the execution environment
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ env.setBufferTimeout(1000);
+
+ // get input data
+ DataStream<String> streamSource = getTextDataStream(env);
+
+ DataStream<Tuple2<String, Integer>> tweets = streamSource
+ // selecting English tweets and splitting to words
+ .flatMap(new SelectEnglishAndTokenizeFlatMap())
+ // returning (word, 1)
+ .map(new MapFunction<String, Tuple2<String, Integer>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<String, Integer> map(String value) throws Exception {
+ return new Tuple2<String, Integer>(value, 1);
+ }
+ })
+ // group by words and sum their occurence
+ .groupBy(0).sum(1)
+ // select word with maximum occurence
+ .flatMap(new SelectMaxOccurence());
+
+ // emit result
+ if (fileOutput) {
+ tweets.writeAsText(outputPath, 1);
+ } else {
+ tweets.print();
+ }
+
+ // execute program
+ env.execute("Twitter Streaming Example");
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ /**
+ * Makes sentences from English tweets.
+ *
+ * <p>
+ * Implements a string tokenizer that splits sentences into words as a
+ * user-defined FlatMapFunction. The function takes a line (String) and
+ * splits it into multiple pairs in the form of "(word,1)" (Tuple2<String,
+ * Integer>).
+ * </p>
+ */
+ public static class SelectEnglishAndTokenizeFlatMap extends JSONParseFlatMap<String, String> {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Select the language from the incoming JSON text
+ */
+ @Override
+ public void flatMap(String value, Collector<String> out) throws Exception {
+ try {
+ if (getString(value, "lang").equals("en")) {
+ // message of tweet
+ StringTokenizer tokenizer = new StringTokenizer(getString(value, "text"));
+
+ // split the message
+ while (tokenizer.hasMoreTokens()) {
+ String result = tokenizer.nextToken().replaceAll("\\s*", "");
+
+ if (result != null && !result.equals("")) {
+ out.collect(result);
+ }
+ }
+ }
+ } catch (JSONException e) {
+
+ }
+ }
+ }
+
+ /**
+ * Implements a user-defined FlatMapFunction that checks if the current
+ * occurence is higher than the maximum occurence. If so, returns the word
+ * and changes the maximum.
+ */
+ public static class SelectMaxOccurence implements
+ FlatMapFunction<Tuple2<String, Integer>, Tuple2<String, Integer>> {
+ private static final long serialVersionUID = 1L;
+ private Integer maximum;
+
+ public SelectMaxOccurence() {
+ this.maximum = 0;
+ }
+
+ @Override
+ public void flatMap(Tuple2<String, Integer> value, Collector<Tuple2<String, Integer>> out)
+ throws Exception {
+ if ((Integer) value.getField(1) >= maximum) {
+ out.collect(value);
+ maximum = (Integer) value.getField(1);
+ }
+ }
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String textPath;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+ if (args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if (args.length == 2) {
+ textPath = args[0];
+ outputPath = args[1];
+ } else {
+ System.err.println("USAGE:\nTwitterStream <pathToPropertiesFile> <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing TwitterStream example with built-in default data.");
+ System.out.println(" Provide parameters to read input data from a file.");
+ System.out.println(" USAGE: TwitterStream <pathToPropertiesFile>");
+ }
+ return true;
+ }
+
+ private static DataStream<String> getTextDataStream(StreamExecutionEnvironment env) {
+ if (fileOutput) {
+ // read the text file from given input path
+ return env.readTextFile(textPath);
+ } else {
+ // get default test text data
+ return env.fromElements(TwitterStreamData.TEXTS);
+ }
+ }
+}
[03/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
new file mode 100644
index 0000000..d6a9ac0
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.deltafunction.EuclideanDistance;
+import org.apache.flink.streaming.api.windowing.extractor.FieldsFromTuple;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.streaming.api.windowing.helper.Delta;
+import org.apache.flink.util.Collector;
+
+/**
+ * This example gives an impression about how to use delta policies. It also
+ * shows how extractors can be used.
+ */
+public class DeltaExtractExample {
+
+ // *************************************************************************
+ // PROGRAM
+ // *************************************************************************
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ DataStream dstream = env
+ .addSource(new CountingSource())
+ .window(Delta.of(1.2, new EuclideanDistance(new FieldsFromTuple(0, 1)), new Tuple3(
+ 0d, 0d, "foo"))).every(Count.of(2)).reduce(new ConcatStrings());
+
+ // emit result
+ if (fileOutput) {
+ dstream.writeAsText(outputPath, 1);
+ } else {
+ dstream.print();
+ }
+
+ // execute the program
+ env.execute("Delta Extract Example");
+
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ private static class CountingSource implements SourceFunction<Tuple3<Double, Double, String>> {
+ private static final long serialVersionUID = 1L;
+
+ private int counter = 0;
+
+ @Override
+ public void invoke(Collector<Tuple3<Double, Double, String>> collector) throws Exception {
+ while (true) {
+ if (counter > 9999) {
+ counter = 0;
+ }
+ collector.collect(new Tuple3<Double, Double, String>((double) counter,
+ (double) counter + 1, "V" + counter++));
+ }
+ }
+ }
+
+ private static final class ConcatStrings implements
+ ReduceFunction<Tuple3<Double, Double, String>> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple3<Double, Double, String> reduce(Tuple3<Double, Double, String> value1,
+ Tuple3<Double, Double, String> value2) throws Exception {
+ return new Tuple3<Double, Double, String>(value1.f0, value2.f1, value1.f2 + "|"
+ + value2.f2);
+ }
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if (args.length == 1) {
+ outputPath = args[0];
+ } else {
+ System.err.println("Usage: DeltaExtractExample <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing DeltaExtractExample with generated data.");
+ System.out.println(" Provide parameter to write to file.");
+ System.out.println(" Usage: DeltaExtractExample <result path>");
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
new file mode 100644
index 0000000..48783f2
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.util.Collector;
+
+/**
+ * This example uses count based tumbling windowing with multiple eviction
+ * policies at the same time.
+ */
+public class MultiplePoliciesExample {
+
+ // *************************************************************************
+ // PROGRAM
+ // *************************************************************************
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<String> stream = env.addSource(new BasicSource())
+ .groupBy(new KeySelector<String, String>(){
+ private static final long serialVersionUID = 1L;
+ @Override
+ public String getKey(String value) throws Exception {
+ return value;
+ }
+ })
+ .window(Count.of(2))
+ .every(Count.of(3), Count.of(5))
+ .reduceGroup(new Concat());
+
+ // emit result
+ if (fileOutput) {
+ stream.writeAsText(outputPath, 1);
+ } else {
+ stream.print();
+ }
+
+ // execute the program
+ env.execute("Multiple Policies Example");
+ }
+
+ /**
+ * This source function indefinitely provides String inputs for the
+ * topology.
+ */
+ public static final class BasicSource implements SourceFunction<String> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final static String STR_1 = new String("streaming");
+ private final static String STR_2 = new String("flink");
+
+ @Override
+ public void invoke(Collector<String> out) throws Exception {
+ // continuous emit
+ while (true) {
+ out.collect(STR_1);
+ out.collect(STR_2);
+ }
+ }
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ /**
+ * This reduce function does a String concat.
+ */
+ public static final class Concat implements GroupReduceFunction<String, String> {
+
+ /**
+ * Auto generates version ID
+ */
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void reduce(Iterable<String> values, Collector<String> out) throws Exception {
+ String output = "|";
+ for (String v : values) {
+ output = output + v + "|";
+ }
+ out.collect(output);
+ }
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if (args.length == 1) {
+ outputPath = args[0];
+ } else {
+ System.err.println("Usage: MultiplePoliciesExample <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing MultiplePoliciesExample with generated data.");
+ System.out.println(" Provide parameter to write to file.");
+ System.out.println(" Usage: MultiplePoliciesExample <result path>");
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
new file mode 100644
index 0000000..cf03477
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.util.Collector;
+
+/**
+ * This example uses count based sliding windows to illustrate different
+ * possibilities for the realization of sliding windows. Take a look on the code
+ * which is commented out to see different setups.
+ */
+public class SlidingExample {
+
+ // *************************************************************************
+ // PROGRAM
+ // *************************************************************************
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ /*
+ * SIMPLE-EXAMPLE: Use this to always keep the newest 10 elements in the
+ * buffer Resulting windows will have an overlap of 5 elements
+ */
+
+ // DataStream<String> stream = env.addSource(new CountingSource())
+ // .window(Count.of(10))
+ // .every(Count.of(5))
+ // .reduce(new Concat());
+
+ /*
+ * ADVANCED-EXAMPLE: Use this to have the last element of the last
+ * window as first element of the next window while the window size is
+ * always 5
+ */
+
+ DataStream<String> stream = env.addSource(new CountingSource())
+ .window(Count.of(5)
+ .withDelete(4))
+ .every(Count.of(4)
+ .startingAt(-1))
+ .reduce(new Concat());
+
+ // emit result
+ if (fileOutput) {
+ stream.writeAsText(outputPath, 1);
+ } else {
+ stream.print();
+ }
+
+ // execute the program
+ env.execute("Sliding Example");
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ private static final class CountingSource implements SourceFunction<String> {
+ private static final long serialVersionUID = 1L;
+
+ private int counter = 0;
+
+ @Override
+ public void invoke(Collector<String> collector) throws Exception {
+ // continuous emit
+ while (true) {
+ if (counter > 9999) {
+ counter = 0;
+ }
+ collector.collect("V" + counter++);
+ }
+ }
+ }
+
+ /**
+ * This reduce function does a String concat.
+ */
+ private static final class Concat implements ReduceFunction<String> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String reduce(String value1, String value2) throws Exception {
+ return value1 + "|" + value2;
+ }
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if (args.length == 1) {
+ outputPath = args[0];
+ } else {
+ System.err.println("Usage: SlidingExample <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing SlidingExample with generated data.");
+ System.out.println(" Provide parameter to write to file.");
+ System.out.println(" Usage: SlidingExample <result path>");
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
new file mode 100644
index 0000000..622aa82
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.RichSourceFunction;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.streaming.api.windowing.helper.Time;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
+import org.apache.flink.util.Collector;
+
+/**
+ * This example shows the functionality of time based windows. It utilizes the
+ * {@link ActiveTriggerPolicy} implementation in the
+ * {@link ActiveTimeTriggerPolicy}.
+ */
+public class TimeWindowingExample {
+
+ // *************************************************************************
+ // PROGRAM
+ // *************************************************************************
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<Integer> stream = env.addSource(new CountingSourceWithSleep())
+ .window(Count.of(100))
+ .every(Time.of(1000, TimeUnit.MILLISECONDS))
+ .groupBy(new MyKey())
+ .sum(0);
+
+ // emit result
+ if (fileOutput) {
+ stream.writeAsText(outputPath, 1);
+ } else {
+ stream.print();
+ }
+
+ // execute the program
+ env.execute("Time Windowing Example");
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ /**
+ * This data source emit one element every 0.001 sec. The output is an
+ * Integer counting the output elements. As soon as the counter reaches
+ * 10000 it is reset to 0. On each reset the source waits 5 sec. before it
+ * restarts to produce elements.
+ */
+ private static final class CountingSourceWithSleep extends RichSourceFunction<Integer> {
+ private static final long serialVersionUID = 1L;
+
+ private int counter = 0;
+ private transient Random rnd;
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ rnd = new Random();
+ }
+
+ @Override
+ public void invoke(Collector<Integer> collector) throws Exception {
+ // continuous emit
+ while (true) {
+ if (counter > 9999) {
+ System.out.println("Source pauses now!");
+ Thread.sleep(5000);
+ System.out.println("Source continouse with emitting now!");
+ counter = 0;
+ }
+ collector.collect(rnd.nextInt(9) + 1);
+
+ // Wait 0.001 sec. before the next emit. Otherwise the source is
+ // too fast for local tests and you might always see
+ // SUM[k=1..9999](k) as result.
+ Thread.sleep(1);
+ counter++;
+ }
+ }
+ }
+
+ private static final class MyKey implements KeySelector<Integer, Integer> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) throws Exception {
+ if (value < 2) {
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if (args.length == 1) {
+ outputPath = args[0];
+ } else {
+ System.err.println("Usage: TimeWindowingExample <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing TimeWindowingExample with generated data.");
+ System.out.println(" Provide parameter to write to file.");
+ System.out.println(" Usage: TimeWindowingExample <result path>");
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
new file mode 100644
index 0000000..0f5d8eb
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
+import org.apache.flink.streaming.api.windowing.helper.Delta;
+import org.apache.flink.streaming.api.windowing.helper.Time;
+import org.apache.flink.util.Collector;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An example of grouped stream windowing where different eviction and trigger
+ * policies can be used. A source fetches events from cars every 1 sec
+ * containing their id, their current speed (kmh), overall elapsed distance (m)
+ * and a timestamp. The streaming example triggers the top speed of each car
+ * every x meters elapsed for the last y seconds.
+ */
+public class TopSpeedWindowingExample {
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings({ "rawtypes", "serial" })
+ DataStream topSpeeds = env
+ .addSource(CarSource.create(numOfCars))
+ .groupBy(0)
+ .window(Time.of(evictionSec, TimeUnit.SECONDS))
+ .every(Delta.of(triggerMeters,
+ new DeltaFunction<Tuple4<Integer, Integer, Double, Long>>() {
+ @Override
+ public double getDelta(
+ Tuple4<Integer, Integer, Double, Long> oldDataPoint,
+ Tuple4<Integer, Integer, Double, Long> newDataPoint) {
+ return newDataPoint.f2 - oldDataPoint.f2;
+ }
+ }, new Tuple4<Integer, Integer, Double, Long>(0, 0, 0d, 0l))).maxBy(1);
+
+ topSpeeds.print();
+ env.execute("CarTopSpeedWindowingExample");
+ }
+
+ private static class CarSource implements
+ SourceFunction<Tuple4<Integer, Integer, Double, Long>> {
+
+ private static final long serialVersionUID = 1L;
+ private Integer[] speeds;
+ private Double[] distances;
+
+ private Random rand = new Random();
+
+ private CarSource(int numOfCars) {
+ speeds = new Integer[numOfCars];
+ distances = new Double[numOfCars];
+ Arrays.fill(speeds, 50);
+ Arrays.fill(distances, 0d);
+ }
+
+ public static CarSource create(int cars) {
+ return new CarSource(cars);
+ }
+
+ @Override
+ public void invoke(Collector<Tuple4<Integer, Integer, Double, Long>> collector)
+ throws Exception {
+
+ while (true) {
+ Thread.sleep(1000);
+ for (int carId = 0; carId < speeds.length; carId++) {
+ if (rand.nextBoolean()) {
+ speeds[carId] = Math.min(100, speeds[carId] + 5);
+ } else {
+ speeds[carId] = Math.max(0, speeds[carId] - 5);
+ }
+ distances[carId] += speeds[carId] / 3.6d;
+ collector.collect(new Tuple4<Integer, Integer, Double, Long>(carId,
+ speeds[carId], distances[carId], System.currentTimeMillis()));
+ }
+ }
+ }
+ }
+
+ private static int numOfCars = 2;
+ private static int evictionSec = 10;
+ private static double triggerMeters = 50;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ if (args.length == 3) {
+ numOfCars = Integer.valueOf(args[0]);
+ evictionSec = Integer.valueOf(args[1]);
+ triggerMeters = Double.valueOf(args[2]);
+ } else {
+ System.err
+ .println("Usage: TopSpeedWindowingExample <numCars> <evictSec> <triggerMeters>");
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/PojoExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/PojoExample.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/PojoExample.java
new file mode 100644
index 0000000..f377863
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/PojoExample.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.examples.wordcount;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.examples.java.wordcount.util.WordCountData;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.Collector;
+
+/**
+ * This example shows an implementation of WordCount without using the Tuple2
+ * type, but a custom class.
+ *
+ * <p>
+ * Usage: <code>WordCount <text path> <result path></code><br>
+ * If no parameters are provided, the program is run with default data from
+ * {@link WordCountData}.
+ *
+ * <p>
+ * This example shows how to:
+ * <ul>
+ * <li>use POJO data types,
+ * <li>write a simple Flink program,
+ * <li>write and use user-defined functions.
+ * </ul>
+ */
+public class PojoExample {
+
+ // *************************************************************************
+ // PROGRAM
+ // *************************************************************************
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ // set up the execution environment
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ // get input data
+ DataStream<String> text = getTextDataStream(env);
+
+ DataStream<Word> counts =
+ // split up the lines into Word objects
+ text.flatMap(new Tokenizer())
+ // group by the field word and sum up the frequency
+ .groupBy("word").sum("frequency");
+
+ if (fileOutput) {
+ counts.writeAsText(outputPath);
+ } else {
+ counts.print();
+ }
+
+ // execute program
+ env.execute("WordCount Pojo Example");
+ }
+
+ // *************************************************************************
+ // DATA TYPES
+ // *************************************************************************
+
+ /**
+ * This is the POJO (Plain Old Java Object) that is being used for all the
+ * operations. As long as all fields are public or have a getter/setter, the
+ * system can handle them
+ */
+ public static class Word {
+
+ private String word;
+ private Integer frequency;
+
+ public Word() {
+ }
+
+ public Word(String word, int i) {
+ this.word = word;
+ this.frequency = i;
+ }
+
+ public String getWord() {
+ return word;
+ }
+
+ public void setWord(String word) {
+ this.word = word;
+ }
+
+ public Integer getFrequency() {
+ return frequency;
+ }
+
+ public void setFrequency(Integer frequency) {
+ this.frequency = frequency;
+ }
+
+ @Override
+ public String toString() {
+ return "(" + word + ", " + frequency + ")";
+ }
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ /**
+ * Implements the string tokenizer that splits sentences into words as a
+ * user-defined FlatMapFunction. The function takes a line (String) and
+ * splits it into multiple pairs in the form of "(word,1)" (Tuple2<String,
+ * Integer>).
+ */
+ public static final class Tokenizer implements FlatMapFunction<String, Word> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(String value, Collector<Word> out) {
+ // normalize and split the line
+ String[] tokens = value.toLowerCase().split("\\W+");
+
+ // emit the pairs
+ for (String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new Word(token, 1));
+ }
+ }
+ }
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String textPath;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if (args.length == 2) {
+ textPath = args[0];
+ outputPath = args[1];
+ } else {
+ System.err.println("Usage: PojoExample <text path> <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing PojoExample example with built-in default data.");
+ System.out.println(" Provide parameters to read input data from a file.");
+ System.out.println(" Usage: PojoExample <text path> <result path>");
+ }
+ return true;
+ }
+
+ private static DataStream<String> getTextDataStream(StreamExecutionEnvironment env) {
+ if (fileOutput) {
+ // read the text file from given input path
+ return env.readTextFile(textPath);
+ } else {
+ // get default test text data
+ return env.fromElements(WordCountData.WORDS);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/WordCount.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/WordCount.java
new file mode 100644
index 0000000..085fe5f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/WordCount.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.wordcount;
+
+import java.util.StringTokenizer;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.examples.java.wordcount.util.WordCountData;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.Collector;
+
+/**
+ * Implements the "WordCount" program that computes a simple word occurrence
+ * histogram over text files in a streaming fashion.
+ *
+ * <p>
+ * The input is a plain text file with lines separated by newline characters.
+ *
+ * <p>
+ * Usage: <code>WordCount <text path> <result path></code><br>
+ * If no parameters are provided, the program is run with default data from
+ * {@link WordCountData}.
+ *
+ * <p>
+ * This example shows how to:
+ * <ul>
+ * <li>write a simple Flink Streaming program,
+ * <li>use tuple data types,
+ * <li>write and use user-defined functions.
+ * </ul>
+ *
+ */
+public class WordCount {
+
+ // *************************************************************************
+ // PROGRAM
+ // *************************************************************************
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ // set up the execution environment
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ // get input data
+ DataStream<String> text = getTextDataStream(env);
+
+ DataStream<Tuple2<String, Integer>> counts =
+ // split up the lines in pairs (2-tuples) containing: (word,1)
+ text.flatMap(new Tokenizer())
+ // group by the tuple field "0" and sum up tuple field "1"
+ .groupBy(0).sum(1);
+
+ // emit result
+ if (fileOutput) {
+ counts.writeAsText(outputPath, 1);
+ } else {
+ counts.print();
+ }
+
+ // execute program
+ env.execute("Streaming WordCount");
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ /**
+ * Implements the string tokenizer that splits sentences into words as a
+ * user-defined FlatMapFunction. The function takes a line (String) and
+ * splits it into multiple pairs in the form of "(word,1)" (Tuple2<String,
+ * Integer>).
+ */
+ public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(String inTuple, Collector<Tuple2<String, Integer>> out)
+ throws Exception {
+ // tokenize the line
+ StringTokenizer tokenizer = new StringTokenizer(inTuple);
+
+ // emit the pairs
+ while (tokenizer.hasMoreTokens()) {
+ out.collect(new Tuple2<String, Integer>(tokenizer.nextToken(), 1));
+ }
+ }
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String textPath;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if (args.length == 2) {
+ textPath = args[0];
+ outputPath = args[1];
+ } else {
+ System.err.println("Usage: WordCount <text path> <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing WordCount example with built-in default data.");
+ System.out.println(" Provide parameters to read input data from a file.");
+ System.out.println(" Usage: WordCount <text path> <result path>");
+ }
+ return true;
+ }
+
+ private static DataStream<String> getTextDataStream(StreamExecutionEnvironment env) {
+ if (fileOutput) {
+ // read the text file from given input path
+ return env.readTextFile(textPath);
+ } else {
+ // get default test text data
+ return env.fromElements(WordCountData.WORDS);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala
new file mode 100644
index 0000000..b38764c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.scala.examples.socket
+
+import org.apache.flink.streaming.api.scala._
+
+/**
+ * This example shows an implementation of WordCount with data from a text socket.
+ * To run the example make sure that the service providing the text data is already up and running.
+ *
+ * To start an example socket text stream on your local machine run netcat from a command line,
+ * where the parameter specifies the port number:
+ *
+ * {{{
+ * nc -lk 9999
+ * }}}
+ *
+ * Usage:
+ * {{{
+ * SocketTextStreamWordCount <hostname> <port> <output path>
+ * }}}
+ *
+ * This example shows how to:
+ *
+ * - use StreamExecutionEnvironment.socketTextStream
+ * - write a simple Flink Streaming program in scala.
+ * - write and use user-defined functions.
+ */
+object SocketTextStreamWordCount {
+
+ def main(args: Array[String]) {
+ if (!parseParameters(args)) {
+ return
+ }
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+
+ //Create streams for names and ages by mapping the inputs to the corresponding objects
+ val text = env.socketTextStream(hostName, port)
+ val counts = text.flatMap { _.toLowerCase.split("\\W+") filter { _.nonEmpty } }
+ .map { (_, 1) }
+ .groupBy(0)
+ .sum(1)
+
+ if (fileOutput) {
+ counts.writeAsCsv(outputPath, 1)
+ } else {
+ counts print
+ }
+
+ env.execute("Scala SocketTextStreamWordCount Example")
+ }
+
+ private def parseParameters(args: Array[String]): Boolean = {
+ if (args.length == 3) {
+ fileOutput = true
+ hostName = args(0)
+ port = args(1).toInt
+ outputPath = args(2)
+ } else if (args.length == 2) {
+ hostName = args(0)
+ port = args(1).toInt
+ } else {
+ System.err.println("Usage: SocketTextStreamWordCount <hostname> <port> [<output path>]")
+ return false
+ }
+ true
+ }
+
+ private var fileOutput: Boolean = false
+ private var hostName: String = null
+ private var port: Int = 0
+ private var outputPath: String = null
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
new file mode 100644
index 0000000..1091aa3
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.scala.examples.windowing
+
+import java.util.concurrent.TimeUnit._
+
+import org.apache.flink.streaming.api.scala._
+import org.apache.flink.streaming.api.scala.windowing.{Delta, Time}
+
+import scala.Stream._
+import scala.math._
+import scala.util.Random
+
+/**
+ * An example of grouped stream windowing where different eviction and
+ * trigger policies can be used. A source fetches events from cars
+ * every 1 sec containing their id, their current speed (kmh),
+ * overall elapsed distance (m) and a timestamp. The streaming
+ * example triggers the top speed of each car every x meters elapsed
+ * for the last y seconds.
+ */
+object TopSpeedWindowing {
+
+ case class CarEvent(carId: Int, speed: Int, distance: Double, time: Long) extends Serializable
+
+ def main(args: Array[String]) {
+ if (!parseParameters(args)) {
+ return
+ }
+
+ val cars = genCarStream().groupBy("carId")
+ .window(Time.of(evictionSec, SECONDS))
+ .every(Delta.of[CarEvent](triggerMeters,
+ (oldSp,newSp) => newSp.distance-oldSp.distance, CarEvent(0,0,0,0)))
+ .maxBy("speed")
+
+ cars print
+
+ StreamExecutionEnvironment.getExecutionEnvironment.execute("TopSpeedWindowing")
+
+ }
+
+ def genCarStream(): DataStream[CarEvent] = {
+
+ def nextSpeed(carEvent : CarEvent) : CarEvent =
+ {
+ val next =
+ if (Random.nextBoolean) min(100, carEvent.speed + 5) else max(0, carEvent.speed - 5)
+ CarEvent(carEvent.carId, next, carEvent.distance + next/3.6d,System.currentTimeMillis)
+ }
+ def carStream(speeds : Stream[CarEvent]) : Stream[CarEvent] =
+ {
+ Thread.sleep(1000)
+ speeds.append(carStream(speeds.map(nextSpeed)))
+ }
+ carStream(range(0, numOfCars).map(CarEvent(_,50,0,System.currentTimeMillis())))
+ }
+
+ def parseParameters(args: Array[String]): Boolean = {
+ if (args.length > 0) {
+ if (args.length == 3) {
+ numOfCars = args(0).toInt
+ evictionSec = args(1).toInt
+ triggerMeters = args(2).toDouble
+ }
+ else {
+ System.err.println("Usage: TopSpeedWindowing <numCars> <evictSec> <triggerMeters>")
+ false
+ }
+ }
+ true
+ }
+
+ var numOfCars = 2
+ var evictionSec = 10
+ var triggerMeters = 50d
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
new file mode 100644
index 0000000..0b78365
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.scala.examples.windowing
+
+import org.apache.flink.streaming.api.scala._
+
+import scala.Stream._
+import scala.util.Random
+import java.util.concurrent.TimeUnit
+
+object WindowJoin {
+
+ case class Name(id: Long, name: String)
+ case class Age(id: Long, age: Int)
+ case class Person(name: String, age: Long)
+
+ def main(args: Array[String]) {
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+
+ //Create streams for names and ages by mapping the inputs to the corresponding objects
+ val names = env.fromCollection(nameStream).map(x => Name(x._1, x._2))
+ val ages = env.fromCollection(ageStream).map(x => Age(x._1, x._2))
+
+ //Join the two input streams by id on the last 2 seconds every second and create new
+ //Person objects containing both name and age
+ val joined =
+ names.join(ages).onWindow(2, TimeUnit.SECONDS)
+ .every(1, TimeUnit.SECONDS)
+ .where("id")
+ .equalTo("id") { (n, a) => Person(n.name, a.age) }
+
+ joined print
+
+ env.execute("WindowJoin")
+ }
+
+ def nameStream() : Stream[(Long,String)] = {
+ def nameMapper(names: Array[String])(x: Int) : (Long, String) =
+ {
+ if(x%100==0) Thread.sleep(1000)
+ (x, names(Random.nextInt(names.length)))
+ }
+ range(1,10000).map(nameMapper(Array("tom", "jerry", "alice", "bob", "john", "grace")))
+ }
+
+ def ageStream() : Stream[(Long,Int)] = {
+ def ageMapper(x: Int) : (Long, Int) =
+ {
+ if(x%100==0) Thread.sleep(1000)
+ (x, Random.nextInt(90))
+ }
+ range(1,10000).map(ageMapper)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/pom.xml b/flink-staging/flink-streaming/flink-streaming-scala/pom.xml
new file mode 100644
index 0000000..c06fba7
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/pom.xml
@@ -0,0 +1,217 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-streaming-scala</artifactId>
+ <name>flink-streaming-scala</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-scala</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-reflect</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-compiler</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.scalamacros</groupId>
+ <artifactId>quasiquotes_${scala.binary.version}</artifactId>
+ <version>${scala.macros.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm</artifactId>
+ </dependency>
+
+ <!-- guava needs to be in "provided" scope, to make sure it is not included into the jars by the shading -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- Scala Compiler -->
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.1.4</version>
+ <executions>
+ <!-- Run scala compiler in the process-resources phase, so that dependencies on
+ scala classes can be resolved later in the (Java) compile phase -->
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+
+ <!-- Run scala compiler in the process-test-resources phase, so that dependencies on
+ scala classes can be resolved later in the (Java) test-compile phase -->
+ <execution>
+ <id>scala-test-compile</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <jvmArgs>
+ <jvmArg>-Xms128m</jvmArg>
+ <jvmArg>-Xmx512m</jvmArg>
+ </jvmArgs>
+ <compilerPlugins>
+ <compilerPlugin>
+ <groupId>org.scalamacros</groupId>
+ <artifactId>paradise_${scala.version}</artifactId>
+ <version>${scala.macros.version}</version>
+ </compilerPlugin>
+ </compilerPlugins>
+ </configuration>
+ </plugin>
+
+ <!-- Eclipse Integration -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.8</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ <projectnatures>
+ <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
+ <projectnature>org.eclipse.jdt.core.javanature</projectnature>
+ </projectnatures>
+ <buildcommands>
+ <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
+ </buildcommands>
+ <classpathContainers>
+ <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
+ <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
+ </classpathContainers>
+ <excludes>
+ <exclude>org.scala-lang:scala-library</exclude>
+ <exclude>org.scala-lang:scala-compiler</exclude>
+ </excludes>
+ <sourceIncludes>
+ <sourceInclude>**/*.scala</sourceInclude>
+ <sourceInclude>**/*.java</sourceInclude>
+ </sourceIncludes>
+ </configuration>
+ </plugin>
+
+ <!-- Adding scala source directories to build path -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <!-- Add src/main/scala to eclipse build path -->
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ <!-- Add src/test/scala to eclipse build path -->
+ <execution>
+ <id>add-test-source</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/test/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
+ <version>0.5.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <verbose>false</verbose>
+ <failOnViolation>true</failOnViolation>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <failOnWarning>false</failOnWarning>
+ <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
+ <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
+ <configLocation>${project.basedir}/../../../tools/maven/scalastyle-config.xml</configLocation>
+ <outputFile>${project.basedir}/scalastyle-output.xml</outputFile>
+ <outputEncoding>UTF-8</outputEncoding>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java b/flink-staging/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java
new file mode 100644
index 0000000..77d102d
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.streaming.scala;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.typeutils.runtime.TupleSerializerBase;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
+import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
+import org.apache.flink.streaming.api.function.aggregation.SumFunction;
+
+import scala.Product;
+
+public class ScalaStreamingAggregator<IN extends Product> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ TupleSerializerBase<IN> serializer;
+ Object[] fields;
+ int length;
+ int position;
+
+ public ScalaStreamingAggregator(TypeSerializer<IN> serializer, int pos) {
+ this.serializer = (TupleSerializerBase<IN>) serializer;
+ this.length = this.serializer.getArity();
+ this.fields = new Object[this.length];
+ this.position = pos;
+ }
+
+ public class Sum extends AggregationFunction<IN> {
+ private static final long serialVersionUID = 1L;
+ SumFunction sumFunction;
+
+ public Sum(SumFunction func) {
+ super(ScalaStreamingAggregator.this.position);
+ this.sumFunction = func;
+ }
+
+ @Override
+ public IN reduce(IN value1, IN value2) throws Exception {
+ for (int i = 0; i < length; i++) {
+ fields[i] = value2.productElement(i);
+ }
+
+ fields[position] = sumFunction.add(fields[position], value1.productElement(position));
+
+ return serializer.createInstance(fields);
+ }
+ }
+
+ public class ProductComparableAggregator extends ComparableAggregator<IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ public ProductComparableAggregator(AggregationFunction.AggregationType aggregationType,
+ boolean first) {
+ super(ScalaStreamingAggregator.this.position, aggregationType, first);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public IN reduce(IN value1, IN value2) throws Exception {
+ Object v1 = value1.productElement(position);
+ Object v2 = value2.productElement(position);
+
+ int c = comparator.isExtremal((Comparable<Object>) v1, v2);
+
+ if (byAggregate) {
+ if (c == 1) {
+ return value1;
+ }
+ if (first) {
+ if (c == 0) {
+ return value1;
+ }
+ }
+
+ return value2;
+ } else {
+ for (int i = 0; i < length; i++) {
+ fields[i] = value2.productElement(i);
+ }
+
+ if (c == 1) {
+ fields[position] = v1;
+ }
+
+ return serializer.createInstance(fields);
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
new file mode 100644
index 0000000..d60e796
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
@@ -0,0 +1,380 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import java.util
+
+import scala.collection.JavaConversions.asScalaBuffer
+import scala.reflect.ClassTag
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.streaming.api.scala._
+import org.apache.flink.streaming.api.datastream.{ConnectedDataStream => JavaCStream}
+import org.apache.flink.streaming.api.function.co.{ CoFlatMapFunction, CoMapFunction, CoReduceFunction, CoWindowFunction }
+import org.apache.flink.streaming.api.invokable.operator.co.{ CoFlatMapInvokable, CoMapInvokable, CoReduceInvokable }
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.util.Collector
+
+class ConnectedDataStream[IN1, IN2](javaStream: JavaCStream[IN1, IN2]) {
+
+ /**
+ * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
+ * the output to a common type. The transformation calls a
+ * @param fun1 for each element of the first input and
+ * @param fun2 for each element of the second input. Each
+ * CoMapFunction call returns exactly one element.
+ *
+ * The CoMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed { @link DataStream}
+ */
+ def map[R: TypeInformation: ClassTag](fun1: IN1 => R, fun2: IN2 => R):
+ DataStream[R] = {
+ if (fun1 == null || fun2 == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+ val comapper = new CoMapFunction[IN1, IN2, R] {
+ def map1(in1: IN1): R = clean(fun1)(in1)
+ def map2(in2: IN2): R = clean(fun2)(in2)
+ }
+
+ new DataStream(javaStream.addCoFunction("map", implicitly[TypeInformation[R]],
+ new CoMapInvokable[IN1, IN2, R](comapper)))
+ }
+
+ /**
+ * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
+ * the output to a common type. The transformation calls a
+ * {@link CoMapFunction#map1} for each element of the first input and
+ * {@link CoMapFunction#map2} for each element of the second input. Each
+ * CoMapFunction call returns exactly one element. The user can also extend
+ * {@link RichCoMapFunction} to gain access to other features provided by
+ * the {@link RichFuntion} interface.
+ *
+ * @param coMapper
+ * The CoMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed { @link DataStream}
+ */
+ def map[R: TypeInformation: ClassTag](coMapper: CoMapFunction[IN1, IN2, R]):
+ DataStream[R] = {
+ if (coMapper == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+
+ new DataStream(javaStream.addCoFunction("map", implicitly[TypeInformation[R]],
+ new CoMapInvokable[IN1, IN2, R](coMapper)))
+ }
+
+ /**
+ * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
+ * maps the output to a common type. The transformation calls a
+ * {@link CoFlatMapFunction#flatMap1} for each element of the first input
+ * and {@link CoFlatMapFunction#flatMap2} for each element of the second
+ * input. Each CoFlatMapFunction call returns any number of elements
+ * including none. The user can also extend {@link RichFlatMapFunction} to
+ * gain access to other features provided by the {@link RichFuntion}
+ * interface.
+ *
+ * @param coFlatMapper
+ * The CoFlatMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed { @link DataStream}
+ */
+ def flatMap[R: TypeInformation: ClassTag](coFlatMapper: CoFlatMapFunction[IN1, IN2, R]):
+ DataStream[R] = {
+ if (coFlatMapper == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ new DataStream[R](javaStream.addCoFunction("flatMap", implicitly[TypeInformation[R]],
+ new CoFlatMapInvokable[IN1, IN2, R](coFlatMapper)))
+ }
+
+ /**
+ * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
+ * maps the output to a common type. The transformation calls a
+ * @param fun1 for each element of the first input
+ * and @param fun2 for each element of the second
+ * input. Each CoFlatMapFunction call returns any number of elements
+ * including none.
+ *
+ * @return The transformed { @link DataStream}
+ */
+ def flatMap[R: TypeInformation: ClassTag](fun1: (IN1, Collector[R]) => Unit,
+ fun2: (IN2, Collector[R]) => Unit): DataStream[R] = {
+ if (fun1 == null || fun2 == null) {
+ throw new NullPointerException("FlatMap functions must not be null.")
+ }
+ val flatMapper = new CoFlatMapFunction[IN1, IN2, R] {
+ def flatMap1(value: IN1, out: Collector[R]): Unit = clean(fun1)(value, out)
+ def flatMap2(value: IN2, out: Collector[R]): Unit = clean(fun2)(value, out)
+ }
+ flatMap(flatMapper)
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 according to keyPosition1 and keyPosition2. Used for
+ * applying function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param keyPosition1
+ * The field used to compute the hashcode of the elements in the
+ * first input stream.
+ * @param keyPosition2
+ * The field used to compute the hashcode of the elements in the
+ * second input stream.
+ * @return @return The transformed { @link ConnectedDataStream}
+ */
+ def groupBy(keyPosition1: Int, keyPosition2: Int): ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(keyPosition1, keyPosition2)
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 according to keyPositions1 and keyPositions2. Used for
+ * applying function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param keyPositions1
+ * The fields used to group the first input stream.
+ * @param keyPositions2
+ * The fields used to group the second input stream.
+ * @return @return The transformed { @link ConnectedDataStream}
+ */
+ def groupBy(keyPositions1: Array[Int], keyPositions2: Array[Int]):
+ ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(keyPositions1, keyPositions2)
+ }
+
+ /**
+ * GroupBy operation for connected data stream using key expressions. Groups
+ * the elements of input1 and input2 according to field1 and field2. A field
+ * expression is either the name of a public field or a getter method with
+ * parentheses of the {@link DataStream}S underlying type. A dot can be used
+ * to drill down into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field1
+ * The grouping expression for the first input
+ * @param field2
+ * The grouping expression for the second input
+ * @return The grouped { @link ConnectedDataStream}
+ */
+ def groupBy(field1: String, field2: String): ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(field1, field2)
+ }
+
+ /**
+ * GroupBy operation for connected data stream using key expressions. Groups
+ * the elements of input1 and input2 according to fields1 and fields2. A
+ * field expression is either the name of a public field or a getter method
+ * with parentheses of the {@link DataStream}S underlying type. A dot can be
+ * used to drill down into objects, as in {@code "field1.getInnerField2()" }
+ * .
+ *
+ * @param fields1
+ * The grouping expressions for the first input
+ * @param fields2
+ * The grouping expressions for the second input
+ * @return The grouped { @link ConnectedDataStream}
+ */
+ def groupBy(fields1: Array[String], fields2: Array[String]):
+ ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(fields1, fields2)
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 using fun1 and fun2. Used for applying
+ * function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param fun1
+ * The function used for grouping the first input
+ * @param fun2
+ * The function used for grouping the second input
+ * @return @return The transformed { @link ConnectedDataStream}
+ */
+ def groupBy[K: TypeInformation](fun1: IN1 => _, fun2: IN2 => _):
+ ConnectedDataStream[IN1, IN2] = {
+
+ val keyExtractor1 = new KeySelector[IN1, Any] {
+ def getKey(in: IN1) = clean(fun1)(in)
+ }
+ val keyExtractor2 = new KeySelector[IN2, Any] {
+ def getKey(in: IN2) = clean(fun2)(in)
+ }
+
+ javaStream.groupBy(keyExtractor1, keyExtractor2)
+ }
+
+ /**
+ * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
+ * the outputs to a common type. If the {@link ConnectedDataStream} is
+ * batched or windowed then the reduce transformation is applied on every
+ * sliding batch/window of the data stream. If the connected data stream is
+ * grouped then the reducer is applied on every group of elements sharing
+ * the same key. This type of reduce is much faster than reduceGroup since
+ * the reduce function can be applied incrementally.
+ *
+ * @param coReducer
+ * The { @link CoReduceFunction} that will be called for every
+ * element of the inputs.
+ * @return The transformed { @link DataStream}.
+ */
+ def reduce[R: TypeInformation: ClassTag](coReducer: CoReduceFunction[IN1, IN2, R]):
+ DataStream[R] = {
+ if (coReducer == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+
+ new DataStream[R](javaStream.addCoFunction("coReduce", implicitly[TypeInformation[R]],
+ new CoReduceInvokable[IN1, IN2, R](coReducer)))
+ }
+
+ /**
+ * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
+ * the outputs to a common type. If the {@link ConnectedDataStream} is
+ * batched or windowed then the reduce transformation is applied on every
+ * sliding batch/window of the data stream. If the connected data stream is
+ * grouped then the reducer is applied on every group of elements sharing
+ * the same key. This type of reduce is much faster than reduceGroup since
+ * the reduce function can be applied incrementally.
+ *
+ * @param reducer1
+ * @param reducer2
+ * @param mapper1
+ * @param mapper2
+ *
+ * @return The transformed { @link DataStream}.
+ */
+ def reduce[R: TypeInformation: ClassTag](reducer1: (IN1, IN1) => IN1,
+ reducer2: (IN2, IN2) => IN2,mapper1: IN1 => R, mapper2: IN2 => R): DataStream[R] = {
+ if (mapper1 == null || mapper2 == null) {
+ throw new NullPointerException("Map functions must not be null.")
+ }
+ if (reducer1 == null || reducer2 == null) {
+ throw new NullPointerException("Reduce functions must not be null.")
+ }
+
+ val reducer = new CoReduceFunction[IN1, IN2, R] {
+ def reduce1(value1: IN1, value2: IN1): IN1 = clean(reducer1)(value1, value2)
+ def map2(value: IN2): R = clean(mapper2)(value)
+ def reduce2(value1: IN2, value2: IN2): IN2 = clean(reducer2)(value1, value2)
+ def map1(value: IN1): R = clean(mapper1)(value)
+ }
+ reduce(reducer)
+ }
+
+ /**
+ * Applies a CoWindow transformation on the connected DataStreams. The
+ * transformation calls the {@link CoWindowFunction#coWindow} method for for
+ * time aligned windows of the two data streams. System time is used as
+ * default to compute windows.
+ *
+ * @param coWindowFunction
+ * The { @link CoWindowFunction} that will be applied for the time
+ * windows.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ *
+ * @return The transformed { @link DataStream}.
+ */
+ def windowReduce[R: TypeInformation: ClassTag](coWindowFunction:
+ CoWindowFunction[IN1, IN2, R], windowSize: Long, slideInterval: Long) = {
+ if (coWindowFunction == null) {
+ throw new NullPointerException("CoWindow function must no be null")
+ }
+
+ javaStream.windowReduce(coWindowFunction, windowSize, slideInterval)
+ }
+
+ /**
+ * Applies a CoWindow transformation on the connected DataStreams. The
+ * transformation calls the {@link CoWindowFunction#coWindow} method for for
+ * time aligned windows of the two data streams. System time is used as
+ * default to compute windows.
+ *
+ * @param coWindower
+ * The coWindowing function to be applied for the time windows.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ *
+ * @return The transformed { @link DataStream}.
+ */
+ def windowReduce[R: TypeInformation: ClassTag](coWindower: (Seq[IN1], Seq[IN2],
+ Collector[R]) => Unit, windowSize: Long, slideInterval: Long) = {
+ if (coWindower == null) {
+ throw new NullPointerException("CoWindow function must no be null")
+ }
+
+ val coWindowFun = new CoWindowFunction[IN1, IN2, R] {
+ def coWindow(first: util.List[IN1], second: util.List[IN2],
+ out: Collector[R]): Unit = clean(coWindower)(first, second, out)
+ }
+
+ javaStream.windowReduce(coWindowFun, windowSize, slideInterval)
+ }
+
+ /**
+ * Returns the first {@link DataStream}.
+ *
+ * @return The first DataStream.
+ */
+ def getFirst(): DataStream[IN1] = {
+ javaStream.getFirst
+ }
+
+ /**
+ * Returns the second {@link DataStream}.
+ *
+ * @return The second DataStream.
+ */
+ def getSecond(): DataStream[IN2] = {
+ javaStream.getSecond
+ }
+
+ /**
+ * Gets the type of the first input
+ *
+ * @return The type of the first input
+ */
+ def getInputType1(): TypeInformation[IN1] = {
+ javaStream.getInputType1
+ }
+
+ /**
+ * Gets the type of the second input
+ *
+ * @return The type of the second input
+ */
+ def getInputType2(): TypeInformation[IN2] = {
+ javaStream.getInputType2
+ }
+
+}
[50/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java
deleted file mode 100644
index d00dbf7..0000000
--- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.java.io;
-
-
-import org.apache.avro.Schema;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.reflect.ReflectData;
-import org.apache.avro.reflect.ReflectDatumWriter;
-import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.flink.api.common.io.FileOutputFormat;
-import org.apache.flink.core.fs.Path;
-
-import java.io.IOException;
-
-public class AvroOutputFormat<E> extends FileOutputFormat<E> {
-
- private static final long serialVersionUID = 1L;
-
- private final Class<E> avroValueType;
-
- private Schema userDefinedSchema = null;
-
- private transient DataFileWriter<E> dataFileWriter;
-
- public AvroOutputFormat(Path filePath, Class<E> type) {
- super(filePath);
- this.avroValueType = type;
- }
-
- public AvroOutputFormat(Class<E> type) {
- this.avroValueType = type;
- }
-
- @Override
- protected String getDirectoryFileName(int taskNumber) {
- return super.getDirectoryFileName(taskNumber) + ".avro";
- }
-
- public void setSchema(Schema schema) {
- this.userDefinedSchema = schema;
- }
-
- @Override
- public void writeRecord(E record) throws IOException {
- dataFileWriter.append(record);
- }
-
- @Override
- public void open(int taskNumber, int numTasks) throws IOException {
- super.open(taskNumber, numTasks);
-
- DatumWriter<E> datumWriter;
- Schema schema = null;
- if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) {
- datumWriter = new SpecificDatumWriter<E>(avroValueType);
- try {
- schema = ((org.apache.avro.specific.SpecificRecordBase)avroValueType.newInstance()).getSchema();
- } catch (InstantiationException e) {
- throw new RuntimeException(e.getMessage());
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e.getMessage());
- }
- } else {
- datumWriter = new ReflectDatumWriter<E>(avroValueType);
- schema = ReflectData.get().getSchema(avroValueType);
- }
- dataFileWriter = new DataFileWriter<E>(datumWriter);
- if (userDefinedSchema == null) {
- dataFileWriter.create(schema, stream);
- } else {
- dataFileWriter.create(userDefinedSchema, stream);
- }
- }
-
- @Override
- public void close() throws IOException {
- dataFileWriter.flush();
- dataFileWriter.close();
- super.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/assembly/test-assembly.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/assembly/test-assembly.xml b/flink-addons/flink-avro/src/test/assembly/test-assembly.xml
deleted file mode 100644
index 0f4561a..0000000
--- a/flink-addons/flink-avro/src/test/assembly/test-assembly.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<assembly>
- <id>test-jar</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${project.build.testOutputDirectory}</directory>
- <outputDirectory>/</outputDirectory>
- <!--modify/add include to match your package(s) -->
- <includes>
- <include>org/apache/flink/api/avro/testjar/**</include>
- </includes>
- </fileSet>
- </fileSets>
-</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java
deleted file mode 100644
index 0b394e3..0000000
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.avro;
-
-import java.io.File;
-import java.net.InetSocketAddress;
-
-import org.apache.flink.client.program.Client;
-import org.apache.flink.client.program.PackagedProgram;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.test.util.ForkableFlinkMiniCluster;
-import org.junit.Assert;
-import org.junit.Test;
-
-
-public class AvroExternalJarProgramITCase {
-
- private static final String JAR_FILE = "target/maven-test-jar.jar";
-
- private static final String TEST_DATA_FILE = "/testdata.avro";
-
- @Test
- public void testExternalProgram() {
-
- ForkableFlinkMiniCluster testMiniCluster = null;
-
- try {
- Configuration config = new Configuration();
- config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 4);
- testMiniCluster = new ForkableFlinkMiniCluster(config, false);
-
- String jarFile = JAR_FILE;
- String testData = getClass().getResource(TEST_DATA_FILE).toString();
-
- PackagedProgram program = new PackagedProgram(new File(jarFile), new String[] { testData });
-
- Client c = new Client(new InetSocketAddress("localhost", testMiniCluster.getJobManagerRPCPort()),
- new Configuration(), program.getUserCodeClassLoader());
- c.run(program, 4, true);
- }
- catch (Throwable t) {
- System.err.println(t.getMessage());
- t.printStackTrace();
- Assert.fail("Error during the packaged program execution: " + t.getMessage());
- }
- finally {
- if (testMiniCluster != null) {
- try {
- testMiniCluster.stop();
- } catch (Throwable t) {}
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java
deleted file mode 100644
index a8bace3..0000000
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.avro;
-
-import org.junit.Assert;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.avro.file.DataFileReader;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.reflect.ReflectDatumReader;
-import org.apache.avro.specific.SpecificDatumReader;
-import org.apache.flink.api.io.avro.example.User;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.java.io.AvroOutputFormat;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.test.util.JavaProgramTestBase;
-
-@SuppressWarnings("serial")
-public class AvroOutputFormatTest extends JavaProgramTestBase {
-
- public static String outputPath1;
-
- public static String outputPath2;
-
- public static String inputPath;
-
- public static String userData = "alice|1|blue\n" +
- "bob|2|red\n" +
- "john|3|yellow\n" +
- "walt|4|black\n";
-
- @Override
- protected void preSubmit() throws Exception {
- inputPath = createTempFile("user", userData);
- outputPath1 = getTempDirPath("avro_output1");
- outputPath2 = getTempDirPath("avro_output2");
- }
-
-
- @Override
- protected void testProgram() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<String, Integer, String>> input = env.readCsvFile(inputPath)
- .fieldDelimiter("|")
- .types(String.class, Integer.class, String.class);
-
- //output the data with AvroOutputFormat for specific user type
- DataSet<User> specificUser = input.map(new ConvertToUser());
- specificUser.write(new AvroOutputFormat<User>(User.class), outputPath1);
-
- //output the data with AvroOutputFormat for reflect user type
- DataSet<ReflectiveUser> reflectiveUser = specificUser.map(new ConvertToReflective());
- reflectiveUser.write(new AvroOutputFormat<ReflectiveUser>(ReflectiveUser.class), outputPath2);
-
- env.execute();
- }
-
- @Override
- protected void postSubmit() throws Exception {
- //compare result for specific user type
- File [] output1;
- File file1 = asFile(outputPath1);
- if (file1.isDirectory()) {
- output1 = file1.listFiles();
- // check for avro ext in dir.
- for (File avroOutput : output1) {
- Assert.assertTrue("Expect extension '.avro'", avroOutput.toString().endsWith(".avro"));
- }
- } else {
- output1 = new File[] {file1};
- }
- List<String> result1 = new ArrayList<String>();
- DatumReader<User> userDatumReader1 = new SpecificDatumReader<User>(User.class);
- for (File avroOutput : output1) {
-
- DataFileReader<User> dataFileReader1 = new DataFileReader<User>(avroOutput, userDatumReader1);
- while (dataFileReader1.hasNext()) {
- User user = dataFileReader1.next();
- result1.add(user.getName() + "|" + user.getFavoriteNumber() + "|" + user.getFavoriteColor());
- }
- }
- for (String expectedResult : userData.split("\n")) {
- Assert.assertTrue("expected user " + expectedResult + " not found.", result1.contains(expectedResult));
- }
-
- //compare result for reflect user type
- File [] output2;
- File file2 = asFile(outputPath2);
- if (file2.isDirectory()) {
- output2 = file2.listFiles();
- } else {
- output2 = new File[] {file2};
- }
- List<String> result2 = new ArrayList<String>();
- DatumReader<ReflectiveUser> userDatumReader2 = new ReflectDatumReader<ReflectiveUser>(ReflectiveUser.class);
- for (File avroOutput : output2) {
- DataFileReader<ReflectiveUser> dataFileReader2 = new DataFileReader<ReflectiveUser>(avroOutput, userDatumReader2);
- while (dataFileReader2.hasNext()) {
- ReflectiveUser user = dataFileReader2.next();
- result2.add(user.getName() + "|" + user.getFavoriteNumber() + "|" + user.getFavoriteColor());
- }
- }
- for (String expectedResult : userData.split("\n")) {
- Assert.assertTrue("expected user " + expectedResult + " not found.", result2.contains(expectedResult));
- }
-
-
- }
-
-
- public final static class ConvertToUser extends RichMapFunction<Tuple3<String, Integer, String>, User> {
-
- @Override
- public User map(Tuple3<String, Integer, String> value) throws Exception {
- return new User(value.f0, value.f1, value.f2);
- }
- }
-
- public final static class ConvertToReflective extends RichMapFunction<User, ReflectiveUser> {
-
- @Override
- public ReflectiveUser map(User value) throws Exception {
- return new ReflectiveUser(value.getName().toString(), value.getFavoriteNumber(), value.getFavoriteColor().toString());
- }
- }
-
-
- public static class ReflectiveUser {
- private String name;
- private int favoriteNumber;
- private String favoriteColor;
-
- public ReflectiveUser() {}
-
- public ReflectiveUser(String name, int favoriteNumber, String favoriteColor) {
- this.name = name;
- this.favoriteNumber = favoriteNumber;
- this.favoriteColor = favoriteColor;
- }
-
- public String getName() {
- return this.name;
- }
- public String getFavoriteColor() {
- return this.favoriteColor;
- }
- public int getFavoriteNumber() {
- return this.favoriteNumber;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/EncoderDecoderTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/EncoderDecoderTest.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/EncoderDecoderTest.java
deleted file mode 100644
index 0724457..0000000
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/EncoderDecoderTest.java
+++ /dev/null
@@ -1,519 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.avro;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.apache.avro.reflect.ReflectDatumReader;
-import org.apache.avro.reflect.ReflectDatumWriter;
-import org.apache.flink.api.io.avro.generated.Colors;
-import org.apache.flink.api.io.avro.generated.User;
-import org.apache.flink.util.StringUtils;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-/**
- * Tests the {@link DataOutputEncoder} and {@link DataInputDecoder} classes for Avro serialization.
- */
-public class EncoderDecoderTest {
- @Test
- public void testComplexStringsDirecty() {
- try {
- Random rnd = new Random(349712539451944123L);
-
- for (int i = 0; i < 10; i++) {
- String testString = StringUtils.getRandomString(rnd, 10, 100);
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
- {
- DataOutputStream dataOut = new DataOutputStream(baos);
- DataOutputEncoder encoder = new DataOutputEncoder();
- encoder.setOut(dataOut);
-
- encoder.writeString(testString);
- dataOut.flush();
- dataOut.close();
- }
-
- byte[] data = baos.toByteArray();
-
- // deserialize
- {
- ByteArrayInputStream bais = new ByteArrayInputStream(data);
- DataInputStream dataIn = new DataInputStream(bais);
- DataInputDecoder decoder = new DataInputDecoder();
- decoder.setIn(dataIn);
-
- String deserialized = decoder.readString();
-
- assertEquals(testString, deserialized);
- }
- }
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test failed due to an exception: " + e.getMessage());
- }
- }
-
- @Test
- public void testPrimitiveTypes() {
-
- testObjectSerialization(new Boolean(true));
- testObjectSerialization(new Boolean(false));
-
- testObjectSerialization(Byte.valueOf((byte) 0));
- testObjectSerialization(Byte.valueOf((byte) 1));
- testObjectSerialization(Byte.valueOf((byte) -1));
- testObjectSerialization(Byte.valueOf(Byte.MIN_VALUE));
- testObjectSerialization(Byte.valueOf(Byte.MAX_VALUE));
-
- testObjectSerialization(Short.valueOf((short) 0));
- testObjectSerialization(Short.valueOf((short) 1));
- testObjectSerialization(Short.valueOf((short) -1));
- testObjectSerialization(Short.valueOf(Short.MIN_VALUE));
- testObjectSerialization(Short.valueOf(Short.MAX_VALUE));
-
- testObjectSerialization(Integer.valueOf(0));
- testObjectSerialization(Integer.valueOf(1));
- testObjectSerialization(Integer.valueOf(-1));
- testObjectSerialization(Integer.valueOf(Integer.MIN_VALUE));
- testObjectSerialization(Integer.valueOf(Integer.MAX_VALUE));
-
- testObjectSerialization(Long.valueOf(0));
- testObjectSerialization(Long.valueOf(1));
- testObjectSerialization(Long.valueOf(-1));
- testObjectSerialization(Long.valueOf(Long.MIN_VALUE));
- testObjectSerialization(Long.valueOf(Long.MAX_VALUE));
-
- testObjectSerialization(Float.valueOf(0));
- testObjectSerialization(Float.valueOf(1));
- testObjectSerialization(Float.valueOf(-1));
- testObjectSerialization(Float.valueOf((float)Math.E));
- testObjectSerialization(Float.valueOf((float)Math.PI));
- testObjectSerialization(Float.valueOf(Float.MIN_VALUE));
- testObjectSerialization(Float.valueOf(Float.MAX_VALUE));
- testObjectSerialization(Float.valueOf(Float.MIN_NORMAL));
- testObjectSerialization(Float.valueOf(Float.NaN));
- testObjectSerialization(Float.valueOf(Float.NEGATIVE_INFINITY));
- testObjectSerialization(Float.valueOf(Float.POSITIVE_INFINITY));
-
- testObjectSerialization(Double.valueOf(0));
- testObjectSerialization(Double.valueOf(1));
- testObjectSerialization(Double.valueOf(-1));
- testObjectSerialization(Double.valueOf(Math.E));
- testObjectSerialization(Double.valueOf(Math.PI));
- testObjectSerialization(Double.valueOf(Double.MIN_VALUE));
- testObjectSerialization(Double.valueOf(Double.MAX_VALUE));
- testObjectSerialization(Double.valueOf(Double.MIN_NORMAL));
- testObjectSerialization(Double.valueOf(Double.NaN));
- testObjectSerialization(Double.valueOf(Double.NEGATIVE_INFINITY));
- testObjectSerialization(Double.valueOf(Double.POSITIVE_INFINITY));
-
- testObjectSerialization("");
- testObjectSerialization("abcdefg");
- testObjectSerialization("ab\u1535\u0155xyz\u706F");
-
- testObjectSerialization(new SimpleTypes(3637, 54876486548L, (byte) 65, "We're out looking for astronauts", (short) 0x2387, 2.65767523));
- testObjectSerialization(new SimpleTypes(705608724, -1L, (byte) -65, "Serve me the sky with a big slice of lemon", (short) Byte.MIN_VALUE, 0.0000001));
- }
-
- @Test
- public void testArrayTypes() {
- {
- int[] array = new int[] {1, 2, 3, 4, 5};
- testObjectSerialization(array);
- }
- {
- long[] array = new long[] {1, 2, 3, 4, 5};
- testObjectSerialization(array);
- }
- {
- float[] array = new float[] {1, 2, 3, 4, 5};
- testObjectSerialization(array);
- }
- {
- double[] array = new double[] {1, 2, 3, 4, 5};
- testObjectSerialization(array);
- }
- {
- String[] array = new String[] {"Oh", "my", "what", "do", "we", "have", "here", "?"};
- testObjectSerialization(array);
- }
- }
-
- @Test
- public void testEmptyArray() {
- {
- int[] array = new int[0];
- testObjectSerialization(array);
- }
- {
- long[] array = new long[0];
- testObjectSerialization(array);
- }
- {
- float[] array = new float[0];
- testObjectSerialization(array);
- }
- {
- double[] array = new double[0];
- testObjectSerialization(array);
- }
- {
- String[] array = new String[0];
- testObjectSerialization(array);
- }
- }
-
- @Test
- public void testObjects() {
- // simple object containing only primitives
- {
- testObjectSerialization(new Book(976243875L, "The Serialization Odysse", 42));
- }
-
- // object with collection
- {
- ArrayList<String> list = new ArrayList<String>();
- list.add("A");
- list.add("B");
- list.add("C");
- list.add("D");
- list.add("E");
-
- testObjectSerialization(new BookAuthor(976243875L, list, "Arno Nym"));
- }
-
- // object with empty collection
- {
- ArrayList<String> list = new ArrayList<String>();
- testObjectSerialization(new BookAuthor(987654321L, list, "The Saurus"));
- }
- }
-
- @Test
- public void testNestedObjectsWithCollections() {
- testObjectSerialization(new ComplexNestedObject2(true));
- }
-
- @Test
- public void testGeneratedObjectWithNullableFields() {
- List<CharSequence> strings = Arrays.asList(new CharSequence[] { "These", "strings", "should", "be", "recognizable", "as", "a", "meaningful", "sequence" });
- List<Boolean> bools = Arrays.asList(true, true, false, false, true, false, true, true);
- Map<CharSequence, Long> map = new HashMap<CharSequence, Long>();
- map.put("1", 1L);
- map.put("2", 2L);
- map.put("3", 3L);
-
- User user = new User("Freudenreich", 1337, "macintosh gray", 1234567890L, 3.1415926, null, true, strings, bools, null, Colors.GREEN, map);
-
- testObjectSerialization(user);
- }
-
- @Test
- public void testVarLenCountEncoding() {
- try {
- long[] values = new long[] { 0, 1, 2, 3, 4, 0, 574, 45236, 0, 234623462, 23462462346L, 0, 9734028767869761L, 0x7fffffffffffffffL};
-
- // write
- ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
- {
- DataOutputStream dataOut = new DataOutputStream(baos);
-
- for (long val : values) {
- DataOutputEncoder.writeVarLongCount(dataOut, val);
- }
-
- dataOut.flush();
- dataOut.close();
- }
-
- // read
- {
- ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
- DataInputStream dataIn = new DataInputStream(bais);
-
- for (long val : values) {
- long read = DataInputDecoder.readVarLongCount(dataIn);
- assertEquals("Wrong var-len encoded value read.", val, read);
- }
- }
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test failed due to an exception: " + e.getMessage());
- }
- }
-
- private static <X> void testObjectSerialization(X obj) {
-
- try {
-
- // serialize
- ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
- {
- DataOutputStream dataOut = new DataOutputStream(baos);
- DataOutputEncoder encoder = new DataOutputEncoder();
- encoder.setOut(dataOut);
-
- @SuppressWarnings("unchecked")
- Class<X> clazz = (Class<X>) obj.getClass();
- ReflectDatumWriter<X> writer = new ReflectDatumWriter<X>(clazz);
-
- writer.write(obj, encoder);
- dataOut.flush();
- dataOut.close();
- }
-
- byte[] data = baos.toByteArray();
- X result = null;
-
- // deserialize
- {
- ByteArrayInputStream bais = new ByteArrayInputStream(data);
- DataInputStream dataIn = new DataInputStream(bais);
- DataInputDecoder decoder = new DataInputDecoder();
- decoder.setIn(dataIn);
-
- @SuppressWarnings("unchecked")
- Class<X> clazz = (Class<X>) obj.getClass();
- ReflectDatumReader<X> reader = new ReflectDatumReader<X>(clazz);
-
- // create a reuse object if possible, otherwise we have no reuse object
- X reuse = null;
- try {
- @SuppressWarnings("unchecked")
- X test = (X) obj.getClass().newInstance();
- reuse = test;
- } catch (Throwable t) {}
-
- result = reader.read(reuse, decoder);
- }
-
- // check
- final String message = "Deserialized object is not the same as the original";
-
- if (obj.getClass().isArray()) {
- Class<?> clazz = obj.getClass();
- if (clazz == byte[].class) {
- assertArrayEquals(message, (byte[]) obj, (byte[]) result);
- }
- else if (clazz == short[].class) {
- assertArrayEquals(message, (short[]) obj, (short[]) result);
- }
- else if (clazz == int[].class) {
- assertArrayEquals(message, (int[]) obj, (int[]) result);
- }
- else if (clazz == long[].class) {
- assertArrayEquals(message, (long[]) obj, (long[]) result);
- }
- else if (clazz == char[].class) {
- assertArrayEquals(message, (char[]) obj, (char[]) result);
- }
- else if (clazz == float[].class) {
- assertArrayEquals(message, (float[]) obj, (float[]) result, 0.0f);
- }
- else if (clazz == double[].class) {
- assertArrayEquals(message, (double[]) obj, (double[]) result, 0.0);
- } else {
- assertArrayEquals(message, (Object[]) obj, (Object[]) result);
- }
- } else {
- assertEquals(message, obj, result);
- }
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test failed due to an exception: " + e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Test Objects
- // --------------------------------------------------------------------------------------------
-
-
- public static final class SimpleTypes {
-
- private final int iVal;
- private final long lVal;
- private final byte bVal;
- private final String sVal;
- private final short rVal;
- private final double dVal;
-
-
- public SimpleTypes() {
- this(0, 0, (byte) 0, "", (short) 0, 0);
- }
-
- public SimpleTypes(int iVal, long lVal, byte bVal, String sVal, short rVal, double dVal) {
- this.iVal = iVal;
- this.lVal = lVal;
- this.bVal = bVal;
- this.sVal = sVal;
- this.rVal = rVal;
- this.dVal = dVal;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj.getClass() == SimpleTypes.class) {
- SimpleTypes other = (SimpleTypes) obj;
-
- return other.iVal == this.iVal &&
- other.lVal == this.lVal &&
- other.bVal == this.bVal &&
- other.sVal.equals(this.sVal) &&
- other.rVal == this.rVal &&
- other.dVal == this.dVal;
-
- } else {
- return false;
- }
- }
- }
-
- public static class ComplexNestedObject1 {
-
- private double doubleValue;
-
- private List<String> stringList;
-
- public ComplexNestedObject1() {}
-
- public ComplexNestedObject1(int offInit) {
- this.doubleValue = 6293485.6723 + offInit;
-
- this.stringList = new ArrayList<String>();
- this.stringList.add("A" + offInit);
- this.stringList.add("somewhat" + offInit);
- this.stringList.add("random" + offInit);
- this.stringList.add("collection" + offInit);
- this.stringList.add("of" + offInit);
- this.stringList.add("strings" + offInit);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj.getClass() == ComplexNestedObject1.class) {
- ComplexNestedObject1 other = (ComplexNestedObject1) obj;
- return other.doubleValue == this.doubleValue && this.stringList.equals(other.stringList);
- } else {
- return false;
- }
- }
- }
-
- public static class ComplexNestedObject2 {
-
- private long longValue;
-
- private Map<String, ComplexNestedObject1> theMap;
-
- public ComplexNestedObject2() {}
-
- public ComplexNestedObject2(boolean init) {
- this.longValue = 46547;
-
- this.theMap = new HashMap<String, ComplexNestedObject1>();
- this.theMap.put("36354L", new ComplexNestedObject1(43546543));
- this.theMap.put("785611L", new ComplexNestedObject1(45784568));
- this.theMap.put("43L", new ComplexNestedObject1(9876543));
- this.theMap.put("-45687L", new ComplexNestedObject1(7897615));
- this.theMap.put("1919876876896L", new ComplexNestedObject1(27154));
- this.theMap.put("-868468468L", new ComplexNestedObject1(546435));
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj.getClass() == ComplexNestedObject2.class) {
- ComplexNestedObject2 other = (ComplexNestedObject2) obj;
- return other.longValue == this.longValue && this.theMap.equals(other.theMap);
- } else {
- return false;
- }
- }
- }
-
- public static class Book {
-
- private long bookId;
- private String title;
- private long authorId;
-
- public Book() {}
-
- public Book(long bookId, String title, long authorId) {
- this.bookId = bookId;
- this.title = title;
- this.authorId = authorId;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj.getClass() == Book.class) {
- Book other = (Book) obj;
- return other.bookId == this.bookId && other.authorId == this.authorId && this.title.equals(other.title);
- } else {
- return false;
- }
- }
- }
-
- public static class BookAuthor {
-
- private long authorId;
- private List<String> bookTitles;
- private String authorName;
-
- public BookAuthor() {}
-
- public BookAuthor(long authorId, List<String> bookTitles, String authorName) {
- this.authorId = authorId;
- this.bookTitles = bookTitles;
- this.authorName = authorName;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj.getClass() == BookAuthor.class) {
- BookAuthor other = (BookAuthor) obj;
- return other.authorName.equals(this.authorName) && other.authorId == this.authorId &&
- other.bookTitles.equals(this.bookTitles);
- } else {
- return false;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java
deleted file mode 100644
index 1174786..0000000
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.avro.testjar;
-
-// ================================================================================================
-// This file defines the classes for the AvroExternalJarProgramITCase.
-// The program is exported into src/test/resources/AvroTestProgram.jar.
-//
-// THIS FILE MUST STAY FULLY COMMENTED SUCH THAT THE HERE DEFINED CLASSES ARE NOT COMPILED
-// AND ADDED TO THE test-classes DIRECTORY. OTHERWISE, THE EXTERNAL CLASS LOADING WILL
-// NOT BE COVERED BY THIS TEST.
-// ================================================================================================
-
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.reflect.ReflectData;
-import org.apache.avro.reflect.ReflectDatumWriter;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.common.functions.RichReduceFunction;
-import org.apache.flink.api.java.io.DiscardingOutputFormat;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.AvroInputFormat;
-import org.apache.flink.core.fs.Path;
-
-public class AvroExternalJarProgram {
-
- public static final class Color {
-
- private String name;
- private double saturation;
-
- public Color() {
- name = "";
- saturation = 1.0;
- }
-
- public Color(String name, double saturation) {
- this.name = name;
- this.saturation = saturation;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public double getSaturation() {
- return saturation;
- }
-
- public void setSaturation(double saturation) {
- this.saturation = saturation;
- }
-
- @Override
- public String toString() {
- return name + '(' + saturation + ')';
- }
- }
-
- public static final class MyUser {
-
- private String name;
- private List<Color> colors;
-
- public MyUser() {
- name = "unknown";
- colors = new ArrayList<Color>();
- }
-
- public MyUser(String name, List<Color> colors) {
- this.name = name;
- this.colors = colors;
- }
-
- public String getName() {
- return name;
- }
-
- public List<Color> getColors() {
- return colors;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public void setColors(List<Color> colors) {
- this.colors = colors;
- }
-
- @Override
- public String toString() {
- return name + " : " + colors;
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- // --------------------------------------------------------------------------------------------
-
- public static final class NameExtractor extends RichMapFunction<MyUser, Tuple2<String, MyUser>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<String, MyUser> map(MyUser u) {
- String namePrefix = u.getName().substring(0, 1);
- return new Tuple2<String, MyUser>(namePrefix, u);
- }
- }
-
- public static final class NameGrouper extends RichReduceFunction<Tuple2<String, MyUser>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<String, MyUser> reduce(Tuple2<String, MyUser> val1, Tuple2<String, MyUser> val2) {
- return val1;
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Test Data
- // --------------------------------------------------------------------------------------------
-
- public static final class Generator {
-
- private final Random rnd = new Random(2389756789345689276L);
-
- public MyUser nextUser() {
- return randomUser();
- }
-
- private MyUser randomUser() {
-
- int numColors = rnd.nextInt(5);
- ArrayList<Color> colors = new ArrayList<Color>(numColors);
- for (int i = 0; i < numColors; i++) {
- colors.add(new Color(randomString(), rnd.nextDouble()));
- }
-
- return new MyUser(randomString(), colors);
- }
-
- private String randomString() {
- char[] c = new char[this.rnd.nextInt(20) + 5];
-
- for (int i = 0; i < c.length; i++) {
- c[i] = (char) (this.rnd.nextInt(150) + 40);
- }
-
- return new String(c);
- }
- }
-
- public static void writeTestData(File testFile, int numRecords) throws IOException {
-
- DatumWriter<MyUser> userDatumWriter = new ReflectDatumWriter<MyUser>(MyUser.class);
- DataFileWriter<MyUser> dataFileWriter = new DataFileWriter<MyUser>(userDatumWriter);
-
- dataFileWriter.create(ReflectData.get().getSchema(MyUser.class), testFile);
-
-
- Generator generator = new Generator();
-
- for (int i = 0; i < numRecords; i++) {
- MyUser user = generator.nextUser();
- dataFileWriter.append(user);
- }
-
- dataFileWriter.close();
- }
-
-// public static void main(String[] args) throws Exception {
-// String testDataFile = new File("src/test/resources/testdata.avro").getAbsolutePath();
-// writeTestData(new File(testDataFile), 50);
-// }
-
- public static void main(String[] args) throws Exception {
- String inputPath = args[0];
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<MyUser> input = env.createInput(new AvroInputFormat<MyUser>(new Path(inputPath), MyUser.class));
-
- DataSet<Tuple2<String, MyUser>> result = input.map(new NameExtractor()).groupBy(0).reduce(new NameGrouper());
-
- result.output(new DiscardingOutputFormat<Tuple2<String,MyUser>>());
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
deleted file mode 100644
index d8d8b46..0000000
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.io.avro;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.avro.util.Utf8;
-import org.apache.flink.api.io.avro.generated.Colors;
-import org.apache.flink.api.io.avro.generated.User;
-import org.apache.flink.api.java.io.AvroInputFormat;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.fs.FileInputSplit;
-import org.apache.flink.core.fs.Path;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test the avro input format.
- * (The testcase is mostly the getting started tutorial of avro)
- * http://avro.apache.org/docs/current/gettingstartedjava.html
- */
-public class AvroRecordInputFormatTest {
-
- private File testFile;
-
- final static String TEST_NAME = "Alyssa";
-
- final static String TEST_ARRAY_STRING_1 = "ELEMENT 1";
- final static String TEST_ARRAY_STRING_2 = "ELEMENT 2";
-
- final static boolean TEST_ARRAY_BOOLEAN_1 = true;
- final static boolean TEST_ARRAY_BOOLEAN_2 = false;
-
- final static Colors TEST_ENUM_COLOR = Colors.GREEN;
-
- final static String TEST_MAP_KEY1 = "KEY 1";
- final static long TEST_MAP_VALUE1 = 8546456L;
- final static String TEST_MAP_KEY2 = "KEY 2";
- final static long TEST_MAP_VALUE2 = 17554L;
-
- @Before
- public void createFiles() throws IOException {
- testFile = File.createTempFile("AvroInputFormatTest", null);
-
- ArrayList<CharSequence> stringArray = new ArrayList<CharSequence>();
- stringArray.add(TEST_ARRAY_STRING_1);
- stringArray.add(TEST_ARRAY_STRING_2);
-
- ArrayList<Boolean> booleanArray = new ArrayList<Boolean>();
- booleanArray.add(TEST_ARRAY_BOOLEAN_1);
- booleanArray.add(TEST_ARRAY_BOOLEAN_2);
-
- HashMap<CharSequence, Long> longMap = new HashMap<CharSequence, Long>();
- longMap.put(TEST_MAP_KEY1, TEST_MAP_VALUE1);
- longMap.put(TEST_MAP_KEY2, TEST_MAP_VALUE2);
-
-
- User user1 = new User();
- user1.setName(TEST_NAME);
- user1.setFavoriteNumber(256);
- user1.setTypeDoubleTest(123.45d);
- user1.setTypeBoolTest(true);
- user1.setTypeArrayString(stringArray);
- user1.setTypeArrayBoolean(booleanArray);
- user1.setTypeEnum(TEST_ENUM_COLOR);
- user1.setTypeMap(longMap);
-
- // Construct via builder
- User user2 = User.newBuilder()
- .setName("Charlie")
- .setFavoriteColor("blue")
- .setFavoriteNumber(null)
- .setTypeBoolTest(false)
- .setTypeDoubleTest(1.337d)
- .setTypeNullTest(null)
- .setTypeLongTest(1337L)
- .setTypeArrayString(new ArrayList<CharSequence>())
- .setTypeArrayBoolean(new ArrayList<Boolean>())
- .setTypeNullableArray(null)
- .setTypeEnum(Colors.RED)
- .setTypeMap(new HashMap<CharSequence, Long>())
- .build();
- DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
- DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);
- dataFileWriter.create(user1.getSchema(), testFile);
- dataFileWriter.append(user1);
- dataFileWriter.append(user2);
- dataFileWriter.close();
- }
-
- @Test
- public void testDeserialisation() throws IOException {
- Configuration parameters = new Configuration();
-
- AvroInputFormat<User> format = new AvroInputFormat<User>(new Path(testFile.getAbsolutePath()), User.class);
-
- format.configure(parameters);
- FileInputSplit[] splits = format.createInputSplits(1);
- assertEquals(splits.length, 1);
- format.open(splits[0]);
-
- User u = format.nextRecord(null);
- assertNotNull(u);
-
- String name = u.getName().toString();
- assertNotNull("empty record", name);
- assertEquals("name not equal", TEST_NAME, name);
-
- // check arrays
- List<CharSequence> sl = u.getTypeArrayString();
- assertEquals("element 0 not equal", TEST_ARRAY_STRING_1, sl.get(0).toString());
- assertEquals("element 1 not equal", TEST_ARRAY_STRING_2, sl.get(1).toString());
-
- List<Boolean> bl = u.getTypeArrayBoolean();
- assertEquals("element 0 not equal", TEST_ARRAY_BOOLEAN_1, bl.get(0));
- assertEquals("element 1 not equal", TEST_ARRAY_BOOLEAN_2, bl.get(1));
-
- // check enums
- Colors enumValue = u.getTypeEnum();
- assertEquals("enum not equal", TEST_ENUM_COLOR, enumValue);
-
- // check maps
- Map<CharSequence, Long> lm = u.getTypeMap();
- assertEquals("map value of key 1 not equal", TEST_MAP_VALUE1, lm.get(new Utf8(TEST_MAP_KEY1)).longValue());
- assertEquals("map value of key 2 not equal", TEST_MAP_VALUE2, lm.get(new Utf8(TEST_MAP_KEY2)).longValue());
-
- assertFalse("expecting second element", format.reachedEnd());
- assertNotNull("expecting second element", format.nextRecord(u));
-
- assertNull(format.nextRecord(u));
- assertTrue(format.reachedEnd());
-
- format.close();
- }
-
- @After
- public void deleteFiles() {
- testFile.delete();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/Colors.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/Colors.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/Colors.java
deleted file mode 100644
index 58e1f5c..0000000
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/Colors.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/**
- * Autogenerated by Avro
- *
- * DO NOT EDIT DIRECTLY
- */
-package org.apache.flink.api.io.avro.generated;
-@SuppressWarnings("all")
-@org.apache.avro.specific.AvroGenerated
-public enum Colors {
- RED, GREEN, BLUE ;
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"enum\",\"name\":\"Colors\",\"namespace\":\"org.apache.flink.api.io.avro.generated\",\"symbols\":[\"RED\",\"GREEN\",\"BLUE\"]}");
- public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/User.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/User.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/User.java
deleted file mode 100644
index 505857e..0000000
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/User.java
+++ /dev/null
@@ -1,755 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/**
- * Autogenerated by Avro
- *
- * DO NOT EDIT DIRECTLY
- */
-package org.apache.flink.api.io.avro.generated;
-@SuppressWarnings("all")
-@org.apache.avro.specific.AvroGenerated
-public class User extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"org.apache.flink.api.io.avro.generated\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]},{\"name\":\"type_long_test\",\"type\":[\"long\",\"null\"]},{\"name\":\"type_double_test\",\"type\":[\"double\"]},{\"name\":\"type_null_test\",\"type\":[\"null\"]},{\"name\":\"type_bool_test\",\"type\":[\"boolean\"]},{\"name\":\"type_array_string\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"type_array_boolean\",\"type\":{\"type\":\"array\",\"items\":\"boolean\"}},{\"name\":\"type_nullable_array\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"string\"}],\"default\":null},{\"name\":\"type_enum\",\"type\":{\"type\":\"enum\",\"name\":\"Colors\",\"symbols\":[\"RED\",\"GREEN\",\"BLUE\"]}},{\"name\":\"type
_map\",\"type\":{\"type\":\"map\",\"values\":\"long\"}}]}");
- public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
- @Deprecated public java.lang.CharSequence name;
- @Deprecated public java.lang.Integer favorite_number;
- @Deprecated public java.lang.CharSequence favorite_color;
- @Deprecated public java.lang.Long type_long_test;
- @Deprecated public java.lang.Object type_double_test;
- @Deprecated public java.lang.Object type_null_test;
- @Deprecated public java.lang.Object type_bool_test;
- @Deprecated public java.util.List<java.lang.CharSequence> type_array_string;
- @Deprecated public java.util.List<java.lang.Boolean> type_array_boolean;
- @Deprecated public java.util.List<java.lang.CharSequence> type_nullable_array;
- @Deprecated public org.apache.flink.api.io.avro.generated.Colors type_enum;
- @Deprecated public java.util.Map<java.lang.CharSequence,java.lang.Long> type_map;
-
- /**
- * Default constructor. Note that this does not initialize fields
- * to their default values from the schema. If that is desired then
- * one should use {@link \#newBuilder()}.
- */
- public User() {}
-
- /**
- * All-args constructor.
- */
- public User(java.lang.CharSequence name, java.lang.Integer favorite_number, java.lang.CharSequence favorite_color, java.lang.Long type_long_test, java.lang.Object type_double_test, java.lang.Object type_null_test, java.lang.Object type_bool_test, java.util.List<java.lang.CharSequence> type_array_string, java.util.List<java.lang.Boolean> type_array_boolean, java.util.List<java.lang.CharSequence> type_nullable_array, org.apache.flink.api.io.avro.generated.Colors type_enum, java.util.Map<java.lang.CharSequence,java.lang.Long> type_map) {
- this.name = name;
- this.favorite_number = favorite_number;
- this.favorite_color = favorite_color;
- this.type_long_test = type_long_test;
- this.type_double_test = type_double_test;
- this.type_null_test = type_null_test;
- this.type_bool_test = type_bool_test;
- this.type_array_string = type_array_string;
- this.type_array_boolean = type_array_boolean;
- this.type_nullable_array = type_nullable_array;
- this.type_enum = type_enum;
- this.type_map = type_map;
- }
-
- public org.apache.avro.Schema getSchema() { return SCHEMA$; }
- // Used by DatumWriter. Applications should not call.
- public java.lang.Object get(int field$) {
- switch (field$) {
- case 0: return name;
- case 1: return favorite_number;
- case 2: return favorite_color;
- case 3: return type_long_test;
- case 4: return type_double_test;
- case 5: return type_null_test;
- case 6: return type_bool_test;
- case 7: return type_array_string;
- case 8: return type_array_boolean;
- case 9: return type_nullable_array;
- case 10: return type_enum;
- case 11: return type_map;
- default: throw new org.apache.avro.AvroRuntimeException("Bad index");
- }
- }
- // Used by DatumReader. Applications should not call.
- @SuppressWarnings(value="unchecked")
- public void put(int field$, java.lang.Object value$) {
- switch (field$) {
- case 0: name = (java.lang.CharSequence)value$; break;
- case 1: favorite_number = (java.lang.Integer)value$; break;
- case 2: favorite_color = (java.lang.CharSequence)value$; break;
- case 3: type_long_test = (java.lang.Long)value$; break;
- case 4: type_double_test = (java.lang.Object)value$; break;
- case 5: type_null_test = (java.lang.Object)value$; break;
- case 6: type_bool_test = (java.lang.Object)value$; break;
- case 7: type_array_string = (java.util.List<java.lang.CharSequence>)value$; break;
- case 8: type_array_boolean = (java.util.List<java.lang.Boolean>)value$; break;
- case 9: type_nullable_array = (java.util.List<java.lang.CharSequence>)value$; break;
- case 10: type_enum = (org.apache.flink.api.io.avro.generated.Colors)value$; break;
- case 11: type_map = (java.util.Map<java.lang.CharSequence,java.lang.Long>)value$; break;
- default: throw new org.apache.avro.AvroRuntimeException("Bad index");
- }
- }
-
- /**
- * Gets the value of the 'name' field.
- */
- public java.lang.CharSequence getName() {
- return name;
- }
-
- /**
- * Sets the value of the 'name' field.
- * @param value the value to set.
- */
- public void setName(java.lang.CharSequence value) {
- this.name = value;
- }
-
- /**
- * Gets the value of the 'favorite_number' field.
- */
- public java.lang.Integer getFavoriteNumber() {
- return favorite_number;
- }
-
- /**
- * Sets the value of the 'favorite_number' field.
- * @param value the value to set.
- */
- public void setFavoriteNumber(java.lang.Integer value) {
- this.favorite_number = value;
- }
-
- /**
- * Gets the value of the 'favorite_color' field.
- */
- public java.lang.CharSequence getFavoriteColor() {
- return favorite_color;
- }
-
- /**
- * Sets the value of the 'favorite_color' field.
- * @param value the value to set.
- */
- public void setFavoriteColor(java.lang.CharSequence value) {
- this.favorite_color = value;
- }
-
- /**
- * Gets the value of the 'type_long_test' field.
- */
- public java.lang.Long getTypeLongTest() {
- return type_long_test;
- }
-
- /**
- * Sets the value of the 'type_long_test' field.
- * @param value the value to set.
- */
- public void setTypeLongTest(java.lang.Long value) {
- this.type_long_test = value;
- }
-
- /**
- * Gets the value of the 'type_double_test' field.
- */
- public java.lang.Object getTypeDoubleTest() {
- return type_double_test;
- }
-
- /**
- * Sets the value of the 'type_double_test' field.
- * @param value the value to set.
- */
- public void setTypeDoubleTest(java.lang.Object value) {
- this.type_double_test = value;
- }
-
- /**
- * Gets the value of the 'type_null_test' field.
- */
- public java.lang.Object getTypeNullTest() {
- return type_null_test;
- }
-
- /**
- * Sets the value of the 'type_null_test' field.
- * @param value the value to set.
- */
- public void setTypeNullTest(java.lang.Object value) {
- this.type_null_test = value;
- }
-
- /**
- * Gets the value of the 'type_bool_test' field.
- */
- public java.lang.Object getTypeBoolTest() {
- return type_bool_test;
- }
-
- /**
- * Sets the value of the 'type_bool_test' field.
- * @param value the value to set.
- */
- public void setTypeBoolTest(java.lang.Object value) {
- this.type_bool_test = value;
- }
-
- /**
- * Gets the value of the 'type_array_string' field.
- */
- public java.util.List<java.lang.CharSequence> getTypeArrayString() {
- return type_array_string;
- }
-
- /**
- * Sets the value of the 'type_array_string' field.
- * @param value the value to set.
- */
- public void setTypeArrayString(java.util.List<java.lang.CharSequence> value) {
- this.type_array_string = value;
- }
-
- /**
- * Gets the value of the 'type_array_boolean' field.
- */
- public java.util.List<java.lang.Boolean> getTypeArrayBoolean() {
- return type_array_boolean;
- }
-
- /**
- * Sets the value of the 'type_array_boolean' field.
- * @param value the value to set.
- */
- public void setTypeArrayBoolean(java.util.List<java.lang.Boolean> value) {
- this.type_array_boolean = value;
- }
-
- /**
- * Gets the value of the 'type_nullable_array' field.
- */
- public java.util.List<java.lang.CharSequence> getTypeNullableArray() {
- return type_nullable_array;
- }
-
- /**
- * Sets the value of the 'type_nullable_array' field.
- * @param value the value to set.
- */
- public void setTypeNullableArray(java.util.List<java.lang.CharSequence> value) {
- this.type_nullable_array = value;
- }
-
- /**
- * Gets the value of the 'type_enum' field.
- */
- public org.apache.flink.api.io.avro.generated.Colors getTypeEnum() {
- return type_enum;
- }
-
- /**
- * Sets the value of the 'type_enum' field.
- * @param value the value to set.
- */
- public void setTypeEnum(org.apache.flink.api.io.avro.generated.Colors value) {
- this.type_enum = value;
- }
-
- /**
- * Gets the value of the 'type_map' field.
- */
- public java.util.Map<java.lang.CharSequence,java.lang.Long> getTypeMap() {
- return type_map;
- }
-
- /**
- * Sets the value of the 'type_map' field.
- * @param value the value to set.
- */
- public void setTypeMap(java.util.Map<java.lang.CharSequence,java.lang.Long> value) {
- this.type_map = value;
- }
-
- /** Creates a new User RecordBuilder */
- public static org.apache.flink.api.io.avro.generated.User.Builder newBuilder() {
- return new org.apache.flink.api.io.avro.generated.User.Builder();
- }
-
- /** Creates a new User RecordBuilder by copying an existing Builder */
- public static org.apache.flink.api.io.avro.generated.User.Builder newBuilder(org.apache.flink.api.io.avro.generated.User.Builder other) {
- return new org.apache.flink.api.io.avro.generated.User.Builder(other);
- }
-
- /** Creates a new User RecordBuilder by copying an existing User instance */
- public static org.apache.flink.api.io.avro.generated.User.Builder newBuilder(org.apache.flink.api.io.avro.generated.User other) {
- return new org.apache.flink.api.io.avro.generated.User.Builder(other);
- }
-
- /**
- * RecordBuilder for User instances.
- */
- public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<User>
- implements org.apache.avro.data.RecordBuilder<User> {
-
- private java.lang.CharSequence name;
- private java.lang.Integer favorite_number;
- private java.lang.CharSequence favorite_color;
- private java.lang.Long type_long_test;
- private java.lang.Object type_double_test;
- private java.lang.Object type_null_test;
- private java.lang.Object type_bool_test;
- private java.util.List<java.lang.CharSequence> type_array_string;
- private java.util.List<java.lang.Boolean> type_array_boolean;
- private java.util.List<java.lang.CharSequence> type_nullable_array;
- private org.apache.flink.api.io.avro.generated.Colors type_enum;
- private java.util.Map<java.lang.CharSequence,java.lang.Long> type_map;
-
- /** Creates a new Builder */
- private Builder() {
- super(org.apache.flink.api.io.avro.generated.User.SCHEMA$);
- }
-
- /** Creates a Builder by copying an existing Builder */
- private Builder(org.apache.flink.api.io.avro.generated.User.Builder other) {
- super(other);
- if (isValidValue(fields()[0], other.name)) {
- this.name = data().deepCopy(fields()[0].schema(), other.name);
- fieldSetFlags()[0] = true;
- }
- if (isValidValue(fields()[1], other.favorite_number)) {
- this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);
- fieldSetFlags()[1] = true;
- }
- if (isValidValue(fields()[2], other.favorite_color)) {
- this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);
- fieldSetFlags()[2] = true;
- }
- if (isValidValue(fields()[3], other.type_long_test)) {
- this.type_long_test = data().deepCopy(fields()[3].schema(), other.type_long_test);
- fieldSetFlags()[3] = true;
- }
- if (isValidValue(fields()[4], other.type_double_test)) {
- this.type_double_test = data().deepCopy(fields()[4].schema(), other.type_double_test);
- fieldSetFlags()[4] = true;
- }
- if (isValidValue(fields()[5], other.type_null_test)) {
- this.type_null_test = data().deepCopy(fields()[5].schema(), other.type_null_test);
- fieldSetFlags()[5] = true;
- }
- if (isValidValue(fields()[6], other.type_bool_test)) {
- this.type_bool_test = data().deepCopy(fields()[6].schema(), other.type_bool_test);
- fieldSetFlags()[6] = true;
- }
- if (isValidValue(fields()[7], other.type_array_string)) {
- this.type_array_string = data().deepCopy(fields()[7].schema(), other.type_array_string);
- fieldSetFlags()[7] = true;
- }
- if (isValidValue(fields()[8], other.type_array_boolean)) {
- this.type_array_boolean = data().deepCopy(fields()[8].schema(), other.type_array_boolean);
- fieldSetFlags()[8] = true;
- }
- if (isValidValue(fields()[9], other.type_nullable_array)) {
- this.type_nullable_array = data().deepCopy(fields()[9].schema(), other.type_nullable_array);
- fieldSetFlags()[9] = true;
- }
- if (isValidValue(fields()[10], other.type_enum)) {
- this.type_enum = data().deepCopy(fields()[10].schema(), other.type_enum);
- fieldSetFlags()[10] = true;
- }
- if (isValidValue(fields()[11], other.type_map)) {
- this.type_map = data().deepCopy(fields()[11].schema(), other.type_map);
- fieldSetFlags()[11] = true;
- }
- }
-
- /** Creates a Builder by copying an existing User instance */
- private Builder(org.apache.flink.api.io.avro.generated.User other) {
- super(org.apache.flink.api.io.avro.generated.User.SCHEMA$);
- if (isValidValue(fields()[0], other.name)) {
- this.name = data().deepCopy(fields()[0].schema(), other.name);
- fieldSetFlags()[0] = true;
- }
- if (isValidValue(fields()[1], other.favorite_number)) {
- this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);
- fieldSetFlags()[1] = true;
- }
- if (isValidValue(fields()[2], other.favorite_color)) {
- this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);
- fieldSetFlags()[2] = true;
- }
- if (isValidValue(fields()[3], other.type_long_test)) {
- this.type_long_test = data().deepCopy(fields()[3].schema(), other.type_long_test);
- fieldSetFlags()[3] = true;
- }
- if (isValidValue(fields()[4], other.type_double_test)) {
- this.type_double_test = data().deepCopy(fields()[4].schema(), other.type_double_test);
- fieldSetFlags()[4] = true;
- }
- if (isValidValue(fields()[5], other.type_null_test)) {
- this.type_null_test = data().deepCopy(fields()[5].schema(), other.type_null_test);
- fieldSetFlags()[5] = true;
- }
- if (isValidValue(fields()[6], other.type_bool_test)) {
- this.type_bool_test = data().deepCopy(fields()[6].schema(), other.type_bool_test);
- fieldSetFlags()[6] = true;
- }
- if (isValidValue(fields()[7], other.type_array_string)) {
- this.type_array_string = data().deepCopy(fields()[7].schema(), other.type_array_string);
- fieldSetFlags()[7] = true;
- }
- if (isValidValue(fields()[8], other.type_array_boolean)) {
- this.type_array_boolean = data().deepCopy(fields()[8].schema(), other.type_array_boolean);
- fieldSetFlags()[8] = true;
- }
- if (isValidValue(fields()[9], other.type_nullable_array)) {
- this.type_nullable_array = data().deepCopy(fields()[9].schema(), other.type_nullable_array);
- fieldSetFlags()[9] = true;
- }
- if (isValidValue(fields()[10], other.type_enum)) {
- this.type_enum = data().deepCopy(fields()[10].schema(), other.type_enum);
- fieldSetFlags()[10] = true;
- }
- if (isValidValue(fields()[11], other.type_map)) {
- this.type_map = data().deepCopy(fields()[11].schema(), other.type_map);
- fieldSetFlags()[11] = true;
- }
- }
-
- /** Gets the value of the 'name' field */
- public java.lang.CharSequence getName() {
- return name;
- }
-
- /** Sets the value of the 'name' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setName(java.lang.CharSequence value) {
- validate(fields()[0], value);
- this.name = value;
- fieldSetFlags()[0] = true;
- return this;
- }
-
- /** Checks whether the 'name' field has been set */
- public boolean hasName() {
- return fieldSetFlags()[0];
- }
-
- /** Clears the value of the 'name' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearName() {
- name = null;
- fieldSetFlags()[0] = false;
- return this;
- }
-
- /** Gets the value of the 'favorite_number' field */
- public java.lang.Integer getFavoriteNumber() {
- return favorite_number;
- }
-
- /** Sets the value of the 'favorite_number' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setFavoriteNumber(java.lang.Integer value) {
- validate(fields()[1], value);
- this.favorite_number = value;
- fieldSetFlags()[1] = true;
- return this;
- }
-
- /** Checks whether the 'favorite_number' field has been set */
- public boolean hasFavoriteNumber() {
- return fieldSetFlags()[1];
- }
-
- /** Clears the value of the 'favorite_number' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearFavoriteNumber() {
- favorite_number = null;
- fieldSetFlags()[1] = false;
- return this;
- }
-
- /** Gets the value of the 'favorite_color' field */
- public java.lang.CharSequence getFavoriteColor() {
- return favorite_color;
- }
-
- /** Sets the value of the 'favorite_color' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setFavoriteColor(java.lang.CharSequence value) {
- validate(fields()[2], value);
- this.favorite_color = value;
- fieldSetFlags()[2] = true;
- return this;
- }
-
- /** Checks whether the 'favorite_color' field has been set */
- public boolean hasFavoriteColor() {
- return fieldSetFlags()[2];
- }
-
- /** Clears the value of the 'favorite_color' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearFavoriteColor() {
- favorite_color = null;
- fieldSetFlags()[2] = false;
- return this;
- }
-
- /** Gets the value of the 'type_long_test' field */
- public java.lang.Long getTypeLongTest() {
- return type_long_test;
- }
-
- /** Sets the value of the 'type_long_test' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setTypeLongTest(java.lang.Long value) {
- validate(fields()[3], value);
- this.type_long_test = value;
- fieldSetFlags()[3] = true;
- return this;
- }
-
- /** Checks whether the 'type_long_test' field has been set */
- public boolean hasTypeLongTest() {
- return fieldSetFlags()[3];
- }
-
- /** Clears the value of the 'type_long_test' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearTypeLongTest() {
- type_long_test = null;
- fieldSetFlags()[3] = false;
- return this;
- }
-
- /** Gets the value of the 'type_double_test' field */
- public java.lang.Object getTypeDoubleTest() {
- return type_double_test;
- }
-
- /** Sets the value of the 'type_double_test' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setTypeDoubleTest(java.lang.Object value) {
- validate(fields()[4], value);
- this.type_double_test = value;
- fieldSetFlags()[4] = true;
- return this;
- }
-
- /** Checks whether the 'type_double_test' field has been set */
- public boolean hasTypeDoubleTest() {
- return fieldSetFlags()[4];
- }
-
- /** Clears the value of the 'type_double_test' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearTypeDoubleTest() {
- type_double_test = null;
- fieldSetFlags()[4] = false;
- return this;
- }
-
- /** Gets the value of the 'type_null_test' field */
- public java.lang.Object getTypeNullTest() {
- return type_null_test;
- }
-
- /** Sets the value of the 'type_null_test' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setTypeNullTest(java.lang.Object value) {
- validate(fields()[5], value);
- this.type_null_test = value;
- fieldSetFlags()[5] = true;
- return this;
- }
-
- /** Checks whether the 'type_null_test' field has been set */
- public boolean hasTypeNullTest() {
- return fieldSetFlags()[5];
- }
-
- /** Clears the value of the 'type_null_test' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearTypeNullTest() {
- type_null_test = null;
- fieldSetFlags()[5] = false;
- return this;
- }
-
- /** Gets the value of the 'type_bool_test' field */
- public java.lang.Object getTypeBoolTest() {
- return type_bool_test;
- }
-
- /** Sets the value of the 'type_bool_test' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setTypeBoolTest(java.lang.Object value) {
- validate(fields()[6], value);
- this.type_bool_test = value;
- fieldSetFlags()[6] = true;
- return this;
- }
-
- /** Checks whether the 'type_bool_test' field has been set */
- public boolean hasTypeBoolTest() {
- return fieldSetFlags()[6];
- }
-
- /** Clears the value of the 'type_bool_test' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearTypeBoolTest() {
- type_bool_test = null;
- fieldSetFlags()[6] = false;
- return this;
- }
-
- /** Gets the value of the 'type_array_string' field */
- public java.util.List<java.lang.CharSequence> getTypeArrayString() {
- return type_array_string;
- }
-
- /** Sets the value of the 'type_array_string' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setTypeArrayString(java.util.List<java.lang.CharSequence> value) {
- validate(fields()[7], value);
- this.type_array_string = value;
- fieldSetFlags()[7] = true;
- return this;
- }
-
- /** Checks whether the 'type_array_string' field has been set */
- public boolean hasTypeArrayString() {
- return fieldSetFlags()[7];
- }
-
- /** Clears the value of the 'type_array_string' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearTypeArrayString() {
- type_array_string = null;
- fieldSetFlags()[7] = false;
- return this;
- }
-
- /** Gets the value of the 'type_array_boolean' field */
- public java.util.List<java.lang.Boolean> getTypeArrayBoolean() {
- return type_array_boolean;
- }
-
- /** Sets the value of the 'type_array_boolean' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setTypeArrayBoolean(java.util.List<java.lang.Boolean> value) {
- validate(fields()[8], value);
- this.type_array_boolean = value;
- fieldSetFlags()[8] = true;
- return this;
- }
-
- /** Checks whether the 'type_array_boolean' field has been set */
- public boolean hasTypeArrayBoolean() {
- return fieldSetFlags()[8];
- }
-
- /** Clears the value of the 'type_array_boolean' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearTypeArrayBoolean() {
- type_array_boolean = null;
- fieldSetFlags()[8] = false;
- return this;
- }
-
- /** Gets the value of the 'type_nullable_array' field */
- public java.util.List<java.lang.CharSequence> getTypeNullableArray() {
- return type_nullable_array;
- }
-
- /** Sets the value of the 'type_nullable_array' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setTypeNullableArray(java.util.List<java.lang.CharSequence> value) {
- validate(fields()[9], value);
- this.type_nullable_array = value;
- fieldSetFlags()[9] = true;
- return this;
- }
-
- /** Checks whether the 'type_nullable_array' field has been set */
- public boolean hasTypeNullableArray() {
- return fieldSetFlags()[9];
- }
-
- /** Clears the value of the 'type_nullable_array' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearTypeNullableArray() {
- type_nullable_array = null;
- fieldSetFlags()[9] = false;
- return this;
- }
-
- /** Gets the value of the 'type_enum' field */
- public org.apache.flink.api.io.avro.generated.Colors getTypeEnum() {
- return type_enum;
- }
-
- /** Sets the value of the 'type_enum' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setTypeEnum(org.apache.flink.api.io.avro.generated.Colors value) {
- validate(fields()[10], value);
- this.type_enum = value;
- fieldSetFlags()[10] = true;
- return this;
- }
-
- /** Checks whether the 'type_enum' field has been set */
- public boolean hasTypeEnum() {
- return fieldSetFlags()[10];
- }
-
- /** Clears the value of the 'type_enum' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearTypeEnum() {
- type_enum = null;
- fieldSetFlags()[10] = false;
- return this;
- }
-
- /** Gets the value of the 'type_map' field */
- public java.util.Map<java.lang.CharSequence,java.lang.Long> getTypeMap() {
- return type_map;
- }
-
- /** Sets the value of the 'type_map' field */
- public org.apache.flink.api.io.avro.generated.User.Builder setTypeMap(java.util.Map<java.lang.CharSequence,java.lang.Long> value) {
- validate(fields()[11], value);
- this.type_map = value;
- fieldSetFlags()[11] = true;
- return this;
- }
-
- /** Checks whether the 'type_map' field has been set */
- public boolean hasTypeMap() {
- return fieldSetFlags()[11];
- }
-
- /** Clears the value of the 'type_map' field */
- public org.apache.flink.api.io.avro.generated.User.Builder clearTypeMap() {
- type_map = null;
- fieldSetFlags()[11] = false;
- return this;
- }
-
- @Override
- public User build() {
- try {
- User record = new User();
- record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]);
- record.favorite_number = fieldSetFlags()[1] ? this.favorite_number : (java.lang.Integer) defaultValue(fields()[1]);
- record.favorite_color = fieldSetFlags()[2] ? this.favorite_color : (java.lang.CharSequence) defaultValue(fields()[2]);
- record.type_long_test = fieldSetFlags()[3] ? this.type_long_test : (java.lang.Long) defaultValue(fields()[3]);
- record.type_double_test = fieldSetFlags()[4] ? this.type_double_test : (java.lang.Object) defaultValue(fields()[4]);
- record.type_null_test = fieldSetFlags()[5] ? this.type_null_test : (java.lang.Object) defaultValue(fields()[5]);
- record.type_bool_test = fieldSetFlags()[6] ? this.type_bool_test : (java.lang.Object) defaultValue(fields()[6]);
- record.type_array_string = fieldSetFlags()[7] ? this.type_array_string : (java.util.List<java.lang.CharSequence>) defaultValue(fields()[7]);
- record.type_array_boolean = fieldSetFlags()[8] ? this.type_array_boolean : (java.util.List<java.lang.Boolean>) defaultValue(fields()[8]);
- record.type_nullable_array = fieldSetFlags()[9] ? this.type_nullable_array : (java.util.List<java.lang.CharSequence>) defaultValue(fields()[9]);
- record.type_enum = fieldSetFlags()[10] ? this.type_enum : (org.apache.flink.api.io.avro.generated.Colors) defaultValue(fields()[10]);
- record.type_map = fieldSetFlags()[11] ? this.type_map : (java.util.Map<java.lang.CharSequence,java.lang.Long>) defaultValue(fields()[11]);
- return record;
- } catch (Exception e) {
- throw new org.apache.avro.AvroRuntimeException(e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
deleted file mode 100644
index 23fbab3..0000000
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.java.io;
-
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.typeutils.PojoTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.core.fs.Path;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class AvroInputFormatTypeExtractionTest {
-
- @Test
- public void testTypeExtraction() {
- try {
- InputFormat<MyAvroType, ?> format = new AvroInputFormat<MyAvroType>(new Path("file:///ignore/this/file"), MyAvroType.class);
-
- TypeInformation<?> typeInfoDirect = TypeExtractor.getInputFormatTypes(format);
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<MyAvroType> input = env.createInput(format);
- TypeInformation<?> typeInfoDataSet = input.getType();
-
-
- Assert.assertTrue(typeInfoDirect instanceof PojoTypeInfo);
- Assert.assertTrue(typeInfoDataSet instanceof PojoTypeInfo);
-
- Assert.assertEquals(MyAvroType.class, typeInfoDirect.getTypeClass());
- Assert.assertEquals(MyAvroType.class, typeInfoDataSet.getTypeClass());
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- public static final class MyAvroType {
-
- public String theString;
-
- private double aDouble;
-
- public double getaDouble() {
- return aDouble;
- }
-
- public void setaDouble(double aDouble) {
- this.aDouble = aDouble;
- }
-
- public void setTheString(String theString) {
- this.theString = theString;
- }
-
- public String getTheString() {
- return theString;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/resources/avro/user.avsc
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/resources/avro/user.avsc b/flink-addons/flink-avro/src/test/resources/avro/user.avsc
deleted file mode 100644
index af3cb75..0000000
--- a/flink-addons/flink-avro/src/test/resources/avro/user.avsc
+++ /dev/null
@@ -1,19 +0,0 @@
-
-{"namespace": "org.apache.flink.api.java.record.io.avro.generated",
- "type": "record",
- "name": "User",
- "fields": [
- {"name": "name", "type": "string"},
- {"name": "favorite_number", "type": ["int", "null"]},
- {"name": "favorite_color", "type": ["string", "null"]},
- {"name": "type_long_test", "type": ["long", "null"]},
- {"name": "type_double_test", "type": ["double"]},
- {"name": "type_null_test", "type": ["null"]},
- {"name": "type_bool_test", "type": ["boolean"]},
- {"name": "type_array_string", "type" : {"type" : "array", "items" : "string"}},
- {"name": "type_array_boolean", "type" : {"type" : "array", "items" : "boolean"}},
- {"name": "type_nullable_array", "type": ["null", {"type":"array", "items":"string"}], "default":null},
- {"name": "type_enum", "type": {"type": "enum", "name": "Colors", "symbols" : ["RED", "GREEN", "BLUE"]}},
- {"name": "type_map", "type": {"type": "map", "values": "long"}}
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/resources/log4j-test.properties b/flink-addons/flink-avro/src/test/resources/log4j-test.properties
deleted file mode 100644
index 0b686e5..0000000
--- a/flink-addons/flink-avro/src/test/resources/log4j-test.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# Set root logger level to DEBUG and its only appender to A1.
-log4j.rootLogger=OFF, A1
-
-# A1 is set to be a ConsoleAppender.
-log4j.appender.A1=org.apache.log4j.ConsoleAppender
-
-# A1 uses PatternLayout.
-log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/resources/logback-test.xml b/flink-addons/flink-avro/src/test/resources/logback-test.xml
deleted file mode 100644
index 8b3bb27..0000000
--- a/flink-addons/flink-avro/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
- </encoder>
- </appender>
-
- <root level="WARN">
- <appender-ref ref="STDOUT"/>
- </root>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/test/resources/testdata.avro
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/resources/testdata.avro b/flink-addons/flink-avro/src/test/resources/testdata.avro
deleted file mode 100644
index 45308b9..0000000
Binary files a/flink-addons/flink-avro/src/test/resources/testdata.avro and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/pom.xml b/flink-addons/flink-hadoop-compatibility/pom.xml
deleted file mode 100644
index 59f541c..0000000
--- a/flink-addons/flink-hadoop-compatibility/pom.xml
+++ /dev/null
@@ -1,80 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-addons</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-hadoop-compatibility</artifactId>
- <name>flink-hadoop-compatibility</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-tests</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <profiles>
- <profile>
- <id>hadoop-2</id>
- <activation>
- <property>
- <!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh -->
- <!--hadoop2--><name>!hadoop.profile</name>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-</project>
[46/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java b/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
deleted file mode 100644
index 3cfaeb9..0000000
--- a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.io.jdbc;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.apache.flink.api.common.io.NonParallelInput;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.flink.api.common.io.DefaultInputSplitAssigner;
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.io.statistics.BaseStatistics;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.io.GenericInputSplit;
-import org.apache.flink.core.io.InputSplit;
-import org.apache.flink.core.io.InputSplitAssigner;
-import org.apache.flink.types.NullValue;
-
-/**
- * InputFormat to read data from a database and generate tuples.
- * The InputFormat has to be configured using the supplied InputFormatBuilder.
- *
- * @param <OUT>
- * @see Tuple
- * @see DriverManager
- */
-public class JDBCInputFormat<OUT extends Tuple> implements InputFormat<OUT, InputSplit>, NonParallelInput {
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(JDBCInputFormat.class);
-
- private String username;
- private String password;
- private String drivername;
- private String dbURL;
- private String query;
-
- private transient Connection dbConn;
- private transient Statement statement;
- private transient ResultSet resultSet;
-
- private int[] columnTypes = null;
-
- public JDBCInputFormat() {
- }
-
- @Override
- public void configure(Configuration parameters) {
- }
-
- /**
- * Connects to the source database and executes the query.
- *
- * @param ignored
- * @throws IOException
- */
- @Override
- public void open(InputSplit ignored) throws IOException {
- try {
- establishConnection();
- statement = dbConn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
- resultSet = statement.executeQuery(query);
- } catch (SQLException se) {
- close();
- throw new IllegalArgumentException("open() failed." + se.getMessage(), se);
- } catch (ClassNotFoundException cnfe) {
- throw new IllegalArgumentException("JDBC-Class not found. - " + cnfe.getMessage(), cnfe);
- }
- }
-
- private void establishConnection() throws SQLException, ClassNotFoundException {
- Class.forName(drivername);
- if (username == null) {
- dbConn = DriverManager.getConnection(dbURL);
- } else {
- dbConn = DriverManager.getConnection(dbURL, username, password);
- }
- }
-
- /**
- * Closes all resources used.
- *
- * @throws IOException Indicates that a resource could not be closed.
- */
- @Override
- public void close() throws IOException {
- try {
- resultSet.close();
- } catch (SQLException se) {
- LOG.info("Inputformat couldn't be closed - " + se.getMessage());
- } catch (NullPointerException npe) {
- }
- try {
- statement.close();
- } catch (SQLException se) {
- LOG.info("Inputformat couldn't be closed - " + se.getMessage());
- } catch (NullPointerException npe) {
- }
- try {
- dbConn.close();
- } catch (SQLException se) {
- LOG.info("Inputformat couldn't be closed - " + se.getMessage());
- } catch (NullPointerException npe) {
- }
- }
-
- /**
- * Checks whether all data has been read.
- *
- * @return boolean value indication whether all data has been read.
- * @throws IOException
- */
- @Override
- public boolean reachedEnd() throws IOException {
- try {
- if (resultSet.isLast()) {
- close();
- return true;
- }
- return false;
- } catch (SQLException se) {
- throw new IOException("Couldn't evaluate reachedEnd() - " + se.getMessage(), se);
- }
- }
-
- /**
- * Stores the next resultSet row in a tuple
- *
- * @param tuple
- * @return tuple containing next row
- * @throws java.io.IOException
- */
- @Override
- public OUT nextRecord(OUT tuple) throws IOException {
- try {
- resultSet.next();
- if (columnTypes == null) {
- extractTypes(tuple);
- }
- addValue(tuple);
- return tuple;
- } catch (SQLException se) {
- close();
- throw new IOException("Couldn't read data - " + se.getMessage(), se);
- } catch (NullPointerException npe) {
- close();
- throw new IOException("Couldn't access resultSet", npe);
- }
- }
-
- private void extractTypes(OUT tuple) throws SQLException, IOException {
- ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
- columnTypes = new int[resultSetMetaData.getColumnCount()];
- if (tuple.getArity() != columnTypes.length) {
- close();
- throw new IOException("Tuple size does not match columncount");
- }
- for (int pos = 0; pos < columnTypes.length; pos++) {
- columnTypes[pos] = resultSetMetaData.getColumnType(pos + 1);
- }
- }
-
- /**
- * Enters data value from the current resultSet into a Record.
- *
- * @param reuse Target Record.
- */
- private void addValue(OUT reuse) throws SQLException {
- for (int pos = 0; pos < columnTypes.length; pos++) {
- switch (columnTypes[pos]) {
- case java.sql.Types.NULL:
- reuse.setField(NullValue.getInstance(), pos);
- break;
- case java.sql.Types.BOOLEAN:
- reuse.setField(resultSet.getBoolean(pos + 1), pos);
- break;
- case java.sql.Types.BIT:
- reuse.setField(resultSet.getBoolean(pos + 1), pos);
- break;
- case java.sql.Types.CHAR:
- reuse.setField(resultSet.getString(pos + 1), pos);
- break;
- case java.sql.Types.NCHAR:
- reuse.setField(resultSet.getString(pos + 1), pos);
- break;
- case java.sql.Types.VARCHAR:
- reuse.setField(resultSet.getString(pos + 1), pos);
- break;
- case java.sql.Types.LONGVARCHAR:
- reuse.setField(resultSet.getString(pos + 1), pos);
- break;
- case java.sql.Types.LONGNVARCHAR:
- reuse.setField(resultSet.getString(pos + 1), pos);
- break;
- case java.sql.Types.TINYINT:
- reuse.setField(resultSet.getShort(pos + 1), pos);
- break;
- case java.sql.Types.SMALLINT:
- reuse.setField(resultSet.getShort(pos + 1), pos);
- break;
- case java.sql.Types.BIGINT:
- reuse.setField(resultSet.getLong(pos + 1), pos);
- break;
- case java.sql.Types.INTEGER:
- reuse.setField(resultSet.getInt(pos + 1), pos);
- break;
- case java.sql.Types.FLOAT:
- reuse.setField(resultSet.getDouble(pos + 1), pos);
- break;
- case java.sql.Types.REAL:
- reuse.setField(resultSet.getFloat(pos + 1), pos);
- break;
- case java.sql.Types.DOUBLE:
- reuse.setField(resultSet.getDouble(pos + 1), pos);
- break;
- case java.sql.Types.DECIMAL:
- reuse.setField(resultSet.getBigDecimal(pos + 1).doubleValue(), pos);
- break;
- case java.sql.Types.NUMERIC:
- reuse.setField(resultSet.getBigDecimal(pos + 1).doubleValue(), pos);
- break;
- case java.sql.Types.DATE:
- reuse.setField(resultSet.getDate(pos + 1).toString(), pos);
- break;
- case java.sql.Types.TIME:
- reuse.setField(resultSet.getTime(pos + 1).getTime(), pos);
- break;
- case java.sql.Types.TIMESTAMP:
- reuse.setField(resultSet.getTimestamp(pos + 1).toString(), pos);
- break;
- case java.sql.Types.SQLXML:
- reuse.setField(resultSet.getSQLXML(pos + 1).toString(), pos);
- break;
- default:
- throw new SQLException("Unsupported sql-type [" + columnTypes[pos] + "] on column [" + pos + "]");
-
- // case java.sql.Types.BINARY:
- // case java.sql.Types.VARBINARY:
- // case java.sql.Types.LONGVARBINARY:
- // case java.sql.Types.ARRAY:
- // case java.sql.Types.JAVA_OBJECT:
- // case java.sql.Types.BLOB:
- // case java.sql.Types.CLOB:
- // case java.sql.Types.NCLOB:
- // case java.sql.Types.DATALINK:
- // case java.sql.Types.DISTINCT:
- // case java.sql.Types.OTHER:
- // case java.sql.Types.REF:
- // case java.sql.Types.ROWID:
- // case java.sql.Types.STRUCT:
- }
- }
- }
-
- @Override
- public BaseStatistics getStatistics(BaseStatistics cachedStatistics) throws IOException {
- return cachedStatistics;
- }
-
- @Override
- public InputSplit[] createInputSplits(int minNumSplits) throws IOException {
- GenericInputSplit[] split = {
- new GenericInputSplit(0, 1)
- };
- return split;
- }
-
- @Override
- public InputSplitAssigner getInputSplitAssigner(InputSplit[] inputSplits) {
- return new DefaultInputSplitAssigner(inputSplits);
- }
-
-
- /**
- * A builder used to set parameters to the output format's configuration in a fluent way.
- * @return builder
- */
- public static JDBCInputFormatBuilder buildJDBCInputFormat() {
- return new JDBCInputFormatBuilder();
- }
-
- public static class JDBCInputFormatBuilder {
- private final JDBCInputFormat format;
-
- public JDBCInputFormatBuilder() {
- this.format = new JDBCInputFormat();
- }
-
- public JDBCInputFormatBuilder setUsername(String username) {
- format.username = username;
- return this;
- }
-
- public JDBCInputFormatBuilder setPassword(String password) {
- format.password = password;
- return this;
- }
-
- public JDBCInputFormatBuilder setDrivername(String drivername) {
- format.drivername = drivername;
- return this;
- }
-
- public JDBCInputFormatBuilder setDBUrl(String dbURL) {
- format.dbURL = dbURL;
- return this;
- }
-
- public JDBCInputFormatBuilder setQuery(String query) {
- format.query = query;
- return this;
- }
-
- public JDBCInputFormat finish() {
- if (format.username == null) {
- LOG.info("Username was not supplied separately.");
- }
- if (format.password == null) {
- LOG.info("Password was not supplied separately.");
- }
- if (format.dbURL == null) {
- throw new IllegalArgumentException("No dababase URL supplied.");
- }
- if (format.query == null) {
- throw new IllegalArgumentException("No query suplied");
- }
- if (format.drivername == null) {
- throw new IllegalArgumentException("No driver supplied");
- }
- return format;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java b/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
deleted file mode 100644
index 6771772..0000000
--- a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.io.jdbc;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.flink.api.common.io.OutputFormat;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.configuration.Configuration;
-
-/**
- * OutputFormat to write tuples into a database.
- * The OutputFormat has to be configured using the supplied OutputFormatBuilder.
- *
- * @param <OUT>
- * @see Tuple
- * @see DriverManager
- */
-public class JDBCOutputFormat<OUT extends Tuple> implements OutputFormat<OUT> {
- private static final long serialVersionUID = 1L;
-
- @SuppressWarnings("unused")
- private static final Logger LOG = LoggerFactory.getLogger(JDBCOutputFormat.class);
-
- private String username;
- private String password;
- private String drivername;
- private String dbURL;
- private String query;
- private int batchInterval = 5000;
-
- private Connection dbConn;
- private PreparedStatement upload;
-
- private SupportedTypes[] types = null;
-
- private int batchCount = 0;
-
- public JDBCOutputFormat() {
- }
-
- @Override
- public void configure(Configuration parameters) {
- }
-
- /**
- * Connects to the target database and initializes the prepared statement.
- *
- * @param taskNumber The number of the parallel instance.
- * @throws IOException Thrown, if the output could not be opened due to an
- * I/O problem.
- */
- @Override
- public void open(int taskNumber, int numTasks) throws IOException {
- try {
- establishConnection();
- upload = dbConn.prepareStatement(query);
- } catch (SQLException sqe) {
- close();
- throw new IllegalArgumentException("open() failed:\t!", sqe);
- } catch (ClassNotFoundException cnfe) {
- close();
- throw new IllegalArgumentException("JDBC-Class not found:\t", cnfe);
- }
- }
-
- private void establishConnection() throws SQLException, ClassNotFoundException {
- Class.forName(drivername);
- if (username == null) {
- dbConn = DriverManager.getConnection(dbURL);
- } else {
- dbConn = DriverManager.getConnection(dbURL, username, password);
- }
- }
-
- private enum SupportedTypes {
- BOOLEAN,
- BYTE,
- SHORT,
- INTEGER,
- LONG,
- STRING,
- FLOAT,
- DOUBLE
- }
-
- /**
- * Adds a record to the prepared statement.
- * <p>
- * When this method is called, the output format is guaranteed to be opened.
- *
- * @param tuple The records to add to the output.
- * @throws IOException Thrown, if the records could not be added due to an I/O problem.
- */
- @Override
- public void writeRecord(OUT tuple) throws IOException {
- try {
- if (types == null) {
- extractTypes(tuple);
- }
- addValues(tuple);
- upload.addBatch();
- batchCount++;
- if (batchCount >= batchInterval) {
- upload.executeBatch();
- batchCount = 0;
- }
- } catch (SQLException sqe) {
- close();
- throw new IllegalArgumentException("writeRecord() failed", sqe);
- } catch (IllegalArgumentException iae) {
- close();
- throw new IllegalArgumentException("writeRecord() failed", iae);
- }
- }
-
- private void extractTypes(OUT tuple) {
- types = new SupportedTypes[tuple.getArity()];
- for (int x = 0; x < tuple.getArity(); x++) {
- types[x] = SupportedTypes.valueOf(tuple.getField(x).getClass().getSimpleName().toUpperCase());
- }
- }
-
- private void addValues(OUT tuple) throws SQLException {
- for (int index = 0; index < tuple.getArity(); index++) {
- switch (types[index]) {
- case BOOLEAN:
- upload.setBoolean(index + 1, (Boolean) tuple.getField(index));
- break;
- case BYTE:
- upload.setByte(index + 1, (Byte) tuple.getField(index));
- break;
- case SHORT:
- upload.setShort(index + 1, (Short) tuple.getField(index));
- break;
- case INTEGER:
- upload.setInt(index + 1, (Integer) tuple.getField(index));
- break;
- case LONG:
- upload.setLong(index + 1, (Long) tuple.getField(index));
- break;
- case STRING:
- upload.setString(index + 1, (String) tuple.getField(index));
- break;
- case FLOAT:
- upload.setFloat(index + 1, (Float) tuple.getField(index));
- break;
- case DOUBLE:
- upload.setDouble(index + 1, (Double) tuple.getField(index));
- break;
- }
- }
- }
-
- /**
- * Executes prepared statement and closes all resources of this instance.
- *
- * @throws IOException Thrown, if the input could not be closed properly.
- */
- @Override
- public void close() throws IOException {
- try {
- upload.executeBatch();
- batchCount = 0;
- } catch (SQLException se) {
- throw new IllegalArgumentException("close() failed", se);
- } catch (NullPointerException se) {
- }
- try {
- upload.close();
- } catch (SQLException se) {
- LOG.info("Inputformat couldn't be closed - " + se.getMessage());
- } catch (NullPointerException npe) {
- }
- try {
- dbConn.close();
- } catch (SQLException se) {
- LOG.info("Inputformat couldn't be closed - " + se.getMessage());
- } catch (NullPointerException npe) {
- }
- }
-
- public static JDBCOutputFormatBuilder buildJDBCOutputFormat() {
- return new JDBCOutputFormatBuilder();
- }
-
- public static class JDBCOutputFormatBuilder {
- private final JDBCOutputFormat format;
-
- protected JDBCOutputFormatBuilder() {
- this.format = new JDBCOutputFormat();
- }
-
- public JDBCOutputFormatBuilder setUsername(String username) {
- format.username = username;
- return this;
- }
-
- public JDBCOutputFormatBuilder setPassword(String password) {
- format.password = password;
- return this;
- }
-
- public JDBCOutputFormatBuilder setDrivername(String drivername) {
- format.drivername = drivername;
- return this;
- }
-
- public JDBCOutputFormatBuilder setDBUrl(String dbURL) {
- format.dbURL = dbURL;
- return this;
- }
-
- public JDBCOutputFormatBuilder setQuery(String query) {
- format.query = query;
- return this;
- }
-
- public JDBCOutputFormatBuilder setBatchInterval(int batchInterval) {
- format.batchInterval = batchInterval;
- return this;
- }
-
- /**
- Finalizes the configuration and checks validity.
- @return Configured JDBCOutputFormat
- */
- public JDBCOutputFormat finish() {
- if (format.username == null) {
- LOG.info("Username was not supplied separately.");
- }
- if (format.password == null) {
- LOG.info("Password was not supplied separately.");
- }
- if (format.dbURL == null) {
- throw new IllegalArgumentException("No dababase URL supplied.");
- }
- if (format.query == null) {
- throw new IllegalArgumentException("No query suplied");
- }
- if (format.drivername == null) {
- throw new IllegalArgumentException("No driver supplied");
- }
- return format;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/example/JDBCExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/example/JDBCExample.java b/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/example/JDBCExample.java
deleted file mode 100644
index 7b012ba..0000000
--- a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/example/JDBCExample.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.io.jdbc.example;
-
-import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.DOUBLE_TYPE_INFO;
-import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.INT_TYPE_INFO;
-import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.Statement;
-
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.jdbc.JDBCInputFormat;
-import org.apache.flink.api.java.io.jdbc.JDBCOutputFormat;
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-
-public class JDBCExample {
-
- public static void main(String[] args) throws Exception {
- prepareTestDb();
-
- ExecutionEnvironment environment = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple5> source
- = environment.createInput(JDBCInputFormat.buildJDBCInputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setDBUrl("jdbc:derby:memory:ebookshop")
- .setQuery("select * from books")
- .finish(),
- new TupleTypeInfo(Tuple5.class, INT_TYPE_INFO, STRING_TYPE_INFO, STRING_TYPE_INFO, DOUBLE_TYPE_INFO, INT_TYPE_INFO)
- );
-
- source.output(JDBCOutputFormat.buildJDBCOutputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setDBUrl("jdbc:derby:memory:ebookshop")
- .setQuery("insert into newbooks (id,title,author,price,qty) values (?,?,?,?,?)")
- .finish());
- environment.execute();
- }
-
- private static void prepareTestDb() throws Exception {
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
- Connection conn = DriverManager.getConnection(dbURL);
-
- StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE books (");
- sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
- sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
- sqlQueryBuilder.append("qty INT DEFAULT NULL,");
- sqlQueryBuilder.append("PRIMARY KEY (id))");
-
- Statement stat = conn.createStatement();
- stat.executeUpdate(sqlQueryBuilder.toString());
- stat.close();
-
- sqlQueryBuilder = new StringBuilder("CREATE TABLE newbooks (");
- sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
- sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
- sqlQueryBuilder.append("qty INT DEFAULT NULL,");
- sqlQueryBuilder.append("PRIMARY KEY (id))");
-
- stat = conn.createStatement();
- stat.executeUpdate(sqlQueryBuilder.toString());
- stat.close();
-
- sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
- sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
- sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
- sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
- sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
- sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
-
- stat = conn.createStatement();
- stat.execute(sqlQueryBuilder.toString());
- stat.close();
-
- conn.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormat.java b/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormat.java
deleted file mode 100644
index 3cd295b..0000000
--- a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormat.java
+++ /dev/null
@@ -1,389 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.record.io.jdbc;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.flink.api.common.io.NonParallelInput;
-import org.apache.flink.api.java.record.io.GenericInputFormat;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.types.BooleanValue;
-import org.apache.flink.types.DoubleValue;
-import org.apache.flink.types.FloatValue;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-import org.apache.flink.types.NullValue;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.ShortValue;
-import org.apache.flink.types.StringValue;
-
-/**
- * InputFormat to read data from a database and generate PactReords.
- * The InputFormat has to be configured with the query, and either all
- * connection parameters or a complete database URL.{@link Configuration} The position of a value inside a Record is
- * determined by the table
- * returned.
- *
- * @see Configuration
- * @see Record
- * @see DriverManager
- */
-public class JDBCInputFormat extends GenericInputFormat implements NonParallelInput {
-
- private static final long serialVersionUID = 1L;
-
- @SuppressWarnings("unused")
- private static final Logger LOG = LoggerFactory.getLogger(JDBCInputFormat.class);
-
-
- public final String DRIVER_KEY = "driver";
- public final String USERNAME_KEY = "username";
- public final String PASSWORD_KEY = "password";
- public final String URL_KEY = "url";
- public final String QUERY_KEY = "query";
-
-
- private String username;
- private String password;
- private String driverName;
- private String dbURL;
- private String query;
-
-
- private transient Connection dbConn;
- private transient Statement statement;
- private transient ResultSet resultSet;
-
-
- /**
- * Creates a non-configured JDBCInputFormat. This format has to be
- * configured using configure(configuration).
- */
- public JDBCInputFormat() {}
-
- /**
- * Creates a JDBCInputFormat and configures it.
- *
- * @param driverName
- * JDBC-Drivename
- * @param dbURL
- * Formatted URL containing all connection parameters.
- * @param username
- * @param password
- * @param query
- * Query to execute.
- */
- public JDBCInputFormat(String driverName, String dbURL, String username, String password, String query) {
- this.driverName = driverName;
- this.query = query;
- this.dbURL = dbURL;
- this.username = username;
- this.password = password;
- }
-
- /**
- * Creates a JDBCInputFormat and configures it.
- *
- * @param driverName
- * JDBC-Drivername
- * @param dbURL
- * Formatted URL containing all connection parameters.
- * @param query
- * Query to execute.
- */
- public JDBCInputFormat(String driverName, String dbURL, String query) {
- this(driverName, dbURL, "", "", query);
- }
-
- /**
- * Creates a JDBCInputFormat and configures it.
- *
- * @param parameters
- * Configuration with all connection parameters.
- * @param query
- * Query to execute.
- */
- public JDBCInputFormat(Configuration parameters, String query) {
- this.driverName = parameters.getString(DRIVER_KEY, "");
- this.username = parameters.getString(USERNAME_KEY, "");
- this.password = parameters.getString(PASSWORD_KEY, "");
- this.dbURL = parameters.getString(URL_KEY, "");
- this.query = query;
- }
-
-
- /**
- * Configures this JDBCInputFormat. This includes setting the connection
- * parameters (if necessary), establishing the connection and executing the
- * query.
- *
- * @param parameters
- * Configuration containing all or no parameters.
- */
- @Override
- public void configure(Configuration parameters) {
- boolean needConfigure = isFieldNullOrEmpty(this.query) || isFieldNullOrEmpty(this.dbURL);
- if (needConfigure) {
- this.driverName = parameters.getString(DRIVER_KEY, null);
- this.username = parameters.getString(USERNAME_KEY, null);
- this.password = parameters.getString(PASSWORD_KEY, null);
- this.query = parameters.getString(QUERY_KEY, null);
- this.dbURL = parameters.getString(URL_KEY, null);
- }
-
- try {
- prepareQueryExecution();
- } catch (SQLException e) {
- throw new IllegalArgumentException("Configure failed:\t!", e);
- }
- }
-
- /**
- * Enters data value from the current resultSet into a Record.
- *
- * @param pos
- * Record position to be set.
- * @param type
- * SQL type of the resultSet value.
- * @param record
- * Target Record.
- */
- private void retrieveTypeAndFillRecord(int pos, int type, Record record) throws SQLException,
- NotTransformableSQLFieldException {
- switch (type) {
- case java.sql.Types.NULL:
- record.setField(pos, NullValue.getInstance());
- break;
- case java.sql.Types.BOOLEAN:
- record.setField(pos, new BooleanValue(resultSet.getBoolean(pos + 1)));
- break;
- case java.sql.Types.BIT:
- record.setField(pos, new BooleanValue(resultSet.getBoolean(pos + 1)));
- break;
- case java.sql.Types.CHAR:
- record.setField(pos, new StringValue(resultSet.getString(pos + 1)));
- break;
- case java.sql.Types.NCHAR:
- record.setField(pos, new StringValue(resultSet.getString(pos + 1)));
- break;
- case java.sql.Types.VARCHAR:
- record.setField(pos, new StringValue(resultSet.getString(pos + 1)));
- break;
- case java.sql.Types.LONGVARCHAR:
- record.setField(pos, new StringValue(resultSet.getString(pos + 1)));
- break;
- case java.sql.Types.LONGNVARCHAR:
- record.setField(pos, new StringValue(resultSet.getString(pos + 1)));
- break;
- case java.sql.Types.TINYINT:
- record.setField(pos, new ShortValue(resultSet.getShort(pos + 1)));
- break;
- case java.sql.Types.SMALLINT:
- record.setField(pos, new ShortValue(resultSet.getShort(pos + 1)));
- break;
- case java.sql.Types.BIGINT:
- record.setField(pos, new LongValue(resultSet.getLong(pos + 1)));
- break;
- case java.sql.Types.INTEGER:
- record.setField(pos, new IntValue(resultSet.getInt(pos + 1)));
- break;
- case java.sql.Types.FLOAT:
- record.setField(pos, new DoubleValue(resultSet.getDouble(pos + 1)));
- break;
- case java.sql.Types.REAL:
- record.setField(pos, new FloatValue(resultSet.getFloat(pos + 1)));
- break;
- case java.sql.Types.DOUBLE:
- record.setField(pos, new DoubleValue(resultSet.getDouble(pos + 1)));
- break;
- case java.sql.Types.DECIMAL:
- record.setField(pos, new DoubleValue(resultSet.getBigDecimal(pos + 1).doubleValue()));
- break;
- case java.sql.Types.NUMERIC:
- record.setField(pos, new DoubleValue(resultSet.getBigDecimal(pos + 1).doubleValue()));
- break;
- case java.sql.Types.DATE:
- record.setField(pos, new StringValue(resultSet.getDate(pos + 1).toString()));
- break;
- case java.sql.Types.TIME:
- record.setField(pos, new LongValue(resultSet.getTime(pos + 1).getTime()));
- break;
- case java.sql.Types.TIMESTAMP:
- record.setField(pos, new StringValue(resultSet.getTimestamp(pos + 1).toString()));
- break;
- case java.sql.Types.SQLXML:
- record.setField(pos, new StringValue(resultSet.getSQLXML(pos + 1).toString()));
- break;
- default:
- throw new NotTransformableSQLFieldException("Unknown sql-type [" + type + "]on column [" + pos + "]");
-
- // case java.sql.Types.BINARY:
- // case java.sql.Types.VARBINARY:
- // case java.sql.Types.LONGVARBINARY:
- // case java.sql.Types.ARRAY:
- // case java.sql.Types.JAVA_OBJECT:
- // case java.sql.Types.BLOB:
- // case java.sql.Types.CLOB:
- // case java.sql.Types.NCLOB:
- // case java.sql.Types.DATALINK:
- // case java.sql.Types.DISTINCT:
- // case java.sql.Types.OTHER:
- // case java.sql.Types.REF:
- // case java.sql.Types.ROWID:
- // case java.sql.Types.STRUCT:
- }
- }
-
- private boolean isFieldNullOrEmpty(String field) {
- return (field == null || field.length() == 0);
- }
-
- private void prepareQueryExecution() throws SQLException {
- setClassForDBType();
- prepareCredentialsAndExecute();
- }
-
- /**
- * Loads appropriate JDBC driver.
- *
- * @param dbType
- * Type of the database.
- * @return boolean value, indication whether an appropriate driver could be
- * found.
- */
- private void setClassForDBType() {
- try {
- Class.forName(driverName);
- } catch (ClassNotFoundException cnfe) {
- throw new IllegalArgumentException("JDBC-Class not found:\t" + cnfe.getLocalizedMessage());
- }
- }
-
- private void prepareCredentialsAndExecute() throws SQLException {
- if (isFieldNullOrEmpty(username)) {
- prepareConnection(dbURL);
- } else {
- prepareConnection();
- }
- executeQuery();
- }
-
- /**
- * Establishes a connection to a database.
- *
- * @param dbURL
- * Assembled URL containing all connection parameters.
- * @return boolean value, indicating whether a connection could be
- * established
- */
- private void prepareConnection(String dbURL) throws SQLException {
- dbConn = DriverManager.getConnection(dbURL);
- }
-
- /**
- * Assembles the Database URL and establishes a connection.
- *
- * @param dbType
- * Type of the database.
- * @param username
- * Login username.
- * @param password
- * Login password.
- * @return boolean value, indicating whether a connection could be
- * established
- */
- private void prepareConnection() throws SQLException {
- dbConn = DriverManager.getConnection(dbURL, username, password);
- }
-
- private void executeQuery() throws SQLException {
- statement = dbConn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
- resultSet = statement.executeQuery(this.query);
- }
-
- /**
- * Checks whether all data has been read.
- *
- * @return boolean value indication whether all data has been read.
- */
- @Override
- public boolean reachedEnd() {
- try {
- if (resultSet.isLast()) {
- resultSet.close();
- statement.close();
- dbConn.close();
- return true;
- } else {
- return false;
- }
- } catch (SQLException e) {
- throw new IllegalArgumentException("Couldn't evaluate reachedEnd():\t" + e.getMessage());
- } catch (NullPointerException e) {
- throw new IllegalArgumentException("Couldn't access resultSet:\t" + e.getMessage());
- }
- }
-
- /**
- * Stores the next resultSet row in a Record
- *
- * @param record
- * target Record
- * @return boolean value indicating that the operation was successful
- */
- @Override
- public Record nextRecord(Record record) {
- try {
- resultSet.next();
- ResultSetMetaData rsmd = resultSet.getMetaData();
- int column_count = rsmd.getColumnCount();
- record.setNumFields(column_count);
-
- for (int pos = 0; pos < column_count; pos++) {
- int type = rsmd.getColumnType(pos + 1);
- retrieveTypeAndFillRecord(pos, type, record);
- }
- return record;
- } catch (SQLException e) {
- throw new IllegalArgumentException("Couldn't read data:\t" + e.getMessage());
- } catch (NotTransformableSQLFieldException e) {
- throw new IllegalArgumentException("Couldn't read data because of unknown column sql-type:\t"
- + e.getMessage());
- } catch (NullPointerException e) {
- throw new IllegalArgumentException("Couldn't access resultSet:\t" + e.getMessage());
- }
- }
-
- public static class NotTransformableSQLFieldException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- public NotTransformableSQLFieldException(String message) {
- super(message);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormat.java b/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormat.java
deleted file mode 100644
index 780001a..0000000
--- a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormat.java
+++ /dev/null
@@ -1,359 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.record.io.jdbc;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-
-import org.apache.flink.api.common.io.FileOutputFormat;
-import org.apache.flink.api.common.io.OutputFormat;
-import org.apache.flink.api.java.record.operators.GenericDataSink;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.types.BooleanValue;
-import org.apache.flink.types.ByteValue;
-import org.apache.flink.types.CharValue;
-import org.apache.flink.types.DoubleValue;
-import org.apache.flink.types.FloatValue;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.ShortValue;
-import org.apache.flink.types.StringValue;
-import org.apache.flink.types.Value;
-
-public class JDBCOutputFormat implements OutputFormat<Record> {
- private static final long serialVersionUID = 1L;
-
- private static final int DEFAULT_BATCH_INTERVERAL = 5000;
-
- public static final String DRIVER_KEY = "driver";
- public static final String USERNAME_KEY = "username";
- public static final String PASSWORD_KEY = "password";
- public static final String URL_KEY = "url";
- public static final String QUERY_KEY = "query";
- public static final String FIELD_COUNT_KEY = "fields";
- public static final String FIELD_TYPE_KEY = "type";
- public static final String BATCH_INTERVAL = "batchInt";
-
- private Connection dbConn;
- private PreparedStatement upload;
-
- private String username;
- private String password;
- private String driverName;
- private String dbURL;
-
- private String query;
- private int fieldCount;
- private Class<? extends Value>[] fieldClasses;
-
- /**
- * Variable indicating the current number of insert sets in a batch.
- */
- private int batchCount = 0;
-
- /**
- * Commit interval of batches.
- * High batch interval: faster inserts, more memory required (reduce if OutOfMemoryExceptions occur)
- * low batch interval: slower inserts, less memory.
- */
- private int batchInterval = DEFAULT_BATCH_INTERVERAL;
-
-
- /**
- * Configures this JDBCOutputFormat.
- *
- * @param parameters
- * Configuration containing all parameters.
- */
- @Override
- public void configure(Configuration parameters) {
- this.driverName = parameters.getString(DRIVER_KEY, null);
- this.username = parameters.getString(USERNAME_KEY, null);
- this.password = parameters.getString(PASSWORD_KEY, null);
- this.dbURL = parameters.getString(URL_KEY, null);
- this.query = parameters.getString(QUERY_KEY, null);
- this.fieldCount = parameters.getInteger(FIELD_COUNT_KEY, 0);
- this.batchInterval = parameters.getInteger(BATCH_INTERVAL, DEFAULT_BATCH_INTERVERAL);
-
- @SuppressWarnings("unchecked")
- Class<Value>[] classes = new Class[this.fieldCount];
- this.fieldClasses = classes;
-
- ClassLoader cl = getClass().getClassLoader();
-
- try {
- for (int i = 0; i < this.fieldCount; i++) {
- Class<? extends Value> clazz = parameters.<Value>getClass(FIELD_TYPE_KEY + i, null, cl);
- if (clazz == null) {
- throw new IllegalArgumentException("Invalid configuration for JDBCOutputFormat: "
- + "No type class for parameter " + i);
- }
- this.fieldClasses[i] = clazz;
- }
- }
- catch (ClassNotFoundException e) {
- throw new RuntimeException("Could not load data type classes.", e);
- }
- }
-
- /**
- * Connects to the target database and initializes the prepared statement.
- *
- * @param taskNumber The number of the parallel instance.
- * @throws IOException Thrown, if the output could not be opened due to an
- * I/O problem.
- */
- @Override
- public void open(int taskNumber, int numTasks) throws IOException {
- try {
- establishConnection();
- upload = dbConn.prepareStatement(query);
- } catch (SQLException sqe) {
- throw new IllegalArgumentException("open() failed:\t!", sqe);
- } catch (ClassNotFoundException cnfe) {
- throw new IllegalArgumentException("JDBC-Class not found:\t", cnfe);
- }
- }
-
- private void establishConnection() throws SQLException, ClassNotFoundException {
- Class.forName(driverName);
- if (username == null) {
- dbConn = DriverManager.getConnection(dbURL);
- } else {
- dbConn = DriverManager.getConnection(dbURL, username, password);
- }
- }
-
- /**
- * Adds a record to the prepared statement.
- * <p>
- * When this method is called, the output format is guaranteed to be opened.
- *
- * @param record The records to add to the output.
- * @throws IOException Thrown, if the records could not be added due to an
- * I/O problem.
- */
-
- @Override
- public void writeRecord(Record record) throws IOException {
- try {
- for (int x = 0; x < record.getNumFields(); x++) {
- Value temp = record.getField(x, fieldClasses[x]);
- addValue(x + 1, temp);
- }
- upload.addBatch();
- batchCount++;
- if(batchCount >= batchInterval) {
- upload.executeBatch();
- batchCount = 0;
- }
- } catch (SQLException sqe) {
- throw new IllegalArgumentException("writeRecord() failed:\t", sqe);
- } catch (IllegalArgumentException iae) {
- throw new IllegalArgumentException("writeRecord() failed:\t", iae);
- }
- }
-
- private enum pactType {
- BooleanValue,
- ByteValue,
- CharValue,
- DoubleValue,
- FloatValue,
- IntValue,
- LongValue,
- ShortValue,
- StringValue
- }
-
- private void addValue(int index, Value value) throws SQLException {
- pactType type;
- try {
- type = pactType.valueOf(value.getClass().getSimpleName());
- } catch (IllegalArgumentException iae) {
- throw new IllegalArgumentException("PactType not supported:\t", iae);
- }
- switch (type) {
- case BooleanValue:
- upload.setBoolean(index, ((BooleanValue) value).getValue());
- break;
- case ByteValue:
- upload.setByte(index, ((ByteValue) value).getValue());
- break;
- case CharValue:
- upload.setString(index, String.valueOf(((CharValue) value).getValue()));
- break;
- case DoubleValue:
- upload.setDouble(index, ((DoubleValue) value).getValue());
- break;
- case FloatValue:
- upload.setFloat(index, ((FloatValue) value).getValue());
- break;
- case IntValue:
- upload.setInt(index, ((IntValue) value).getValue());
- break;
- case LongValue:
- upload.setLong(index, ((LongValue) value).getValue());
- break;
- case ShortValue:
- upload.setShort(index, ((ShortValue) value).getValue());
- break;
- case StringValue:
- upload.setString(index, ((StringValue) value).getValue());
- break;
- }
- }
-
- /**
- * Executes prepared statement and closes all resources of this instance.
- *
- * @throws IOException Thrown, if the input could not be closed properly.
- */
- @Override
- public void close() throws IOException {
- try {
- upload.executeBatch();
- batchCount = 0;
- upload.close();
- dbConn.close();
- } catch (SQLException sqe) {
- throw new IllegalArgumentException("close() failed:\t", sqe);
- }
- }
-
- /**
- * Creates a configuration builder that can be used to set the
- * output format's parameters to the config in a fluent fashion.
- *
- * @return A config builder for setting parameters.
- */
- public static ConfigBuilder configureOutputFormat(GenericDataSink target) {
- return new ConfigBuilder(target.getParameters());
- }
-
- /**
- * Abstract builder used to set parameters to the output format's
- * configuration in a fluent way.
- */
- protected static abstract class AbstractConfigBuilder<T>
- extends FileOutputFormat.AbstractConfigBuilder<T> {
-
- /**
- * Creates a new builder for the given configuration.
- *
- * @param config The configuration into which the parameters will be written.
- */
- protected AbstractConfigBuilder(Configuration config) {
- super(config);
- }
-
- /**
- * Sets the query field.
- * @param value value to be set.
- * @return The builder itself.
- */
- public T setQuery(String value) {
- this.config.setString(QUERY_KEY, value);
- @SuppressWarnings("unchecked")
- T ret = (T) this;
- return ret;
- }
-
- /**
- * Sets the url field.
- * @param value value to be set.
- * @return The builder itself.
- */
- public T setUrl(String value) {
- this.config.setString(URL_KEY, value);
- @SuppressWarnings("unchecked")
- T ret = (T) this;
- return ret;
- }
-
- /**
- * Sets the username field.
- * @param value value to be set.
- * @return The builder itself.
- */
- public T setUsername(String value) {
- this.config.setString(USERNAME_KEY, value);
- @SuppressWarnings("unchecked")
- T ret = (T) this;
- return ret;
- }
-
- /**
- * Sets the password field.
- * @param value value to be set.
- * @return The builder itself.
- */
- public T setPassword(String value) {
- this.config.setString(PASSWORD_KEY, value);
- @SuppressWarnings("unchecked")
- T ret = (T) this;
- return ret;
- }
-
- /**
- * Sets the driver field.
- * @param value value to be set.
- * @return The builder itself.
- */
- public T setDriver(String value) {
- this.config.setString(DRIVER_KEY, value);
- @SuppressWarnings("unchecked")
- T ret = (T) this;
- return ret;
- }
-
- /**
- * Sets the type of a column.
- * Types are applied in the order they were set.
- * @param type PactType to apply.
- * @return The builder itself.
- */
- public T setClass(Class<? extends Value> type) {
- final int numYet = this.config.getInteger(FIELD_COUNT_KEY, 0);
- this.config.setClass(FIELD_TYPE_KEY + numYet, type);
- this.config.setInteger(FIELD_COUNT_KEY, numYet + 1);
- @SuppressWarnings("unchecked")
- T ret = (T) this;
- return ret;
- }
- }
-
- /**
- * A builder used to set parameters to the output format's configuration in a fluent way.
- */
- public static final class ConfigBuilder extends AbstractConfigBuilder<ConfigBuilder> {
- /**
- * Creates a new builder for the given configuration.
- *
- * @param targetConfig The configuration into which the parameters will be written.
- */
- protected ConfigBuilder(Configuration targetConfig) {
- super(targetConfig);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/example/JDBCExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/example/JDBCExample.java b/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/example/JDBCExample.java
deleted file mode 100644
index 58aaf9e..0000000
--- a/flink-addons/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/example/JDBCExample.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.record.io.jdbc.example;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.Statement;
-
-import org.apache.flink.api.common.JobExecutionResult;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.Program;
-import org.apache.flink.api.common.ProgramDescription;
-import org.apache.flink.api.java.record.io.jdbc.JDBCInputFormat;
-import org.apache.flink.api.java.record.io.jdbc.JDBCOutputFormat;
-import org.apache.flink.api.java.record.operators.GenericDataSink;
-import org.apache.flink.api.java.record.operators.GenericDataSource;
-import org.apache.flink.client.LocalExecutor;
-import org.apache.flink.types.FloatValue;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.StringValue;
-
-/**
- * Stand-alone example for the JDBC connector.
- *
- * NOTE: To run this example, you need the apache derby code in your classpath.
- * See the Maven file (pom.xml) for a reference to the derby dependency. You can
- * simply Change the scope of the Maven dependency from test to compile.
- */
-public class JDBCExample implements Program, ProgramDescription {
-
- @Override
- public Plan getPlan(String[] args) {
- /*
- * In this example we use the constructor where the url contains all the settings that are needed.
- * You could also use the default constructor and deliver a Configuration with all the needed settings.
- * You also could set the settings to the source-instance.
- */
- GenericDataSource<JDBCInputFormat> source = new GenericDataSource<JDBCInputFormat>(
- new JDBCInputFormat(
- "org.apache.derby.jdbc.EmbeddedDriver",
- "jdbc:derby:memory:ebookshop",
- "select * from books"),
- "Data Source");
-
- GenericDataSink sink = new GenericDataSink(new JDBCOutputFormat(), "Data Output");
- JDBCOutputFormat.configureOutputFormat(sink)
- .setDriver("org.apache.derby.jdbc.EmbeddedDriver")
- .setUrl("jdbc:derby:memory:ebookshop")
- .setQuery("insert into newbooks (id,title,author,price,qty) values (?,?,?,?,?)")
- .setClass(IntValue.class)
- .setClass(StringValue.class)
- .setClass(StringValue.class)
- .setClass(FloatValue.class)
- .setClass(IntValue.class);
-
- sink.addInput(source);
- return new Plan(sink, "JDBC Example Job");
- }
-
- @Override
- public String getDescription() {
- return "Parameter:";
- }
-
- /*
- * To run this example, you need the apache derby code in your classpath!
- */
- public static void main(String[] args) throws Exception {
-
- prepareTestDb();
- JDBCExample tut = new JDBCExample();
- JobExecutionResult res = LocalExecutor.execute(tut, args);
- System.out.println("runtime: " + res.getNetRuntime());
-
- System.exit(0);
- }
-
- private static void prepareTestDb() throws Exception {
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
- Connection conn = DriverManager.getConnection(dbURL);
-
- StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE books (");
- sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
- sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
- sqlQueryBuilder.append("qty INT DEFAULT NULL,");
- sqlQueryBuilder.append("PRIMARY KEY (id))");
-
- Statement stat = conn.createStatement();
- stat.executeUpdate(sqlQueryBuilder.toString());
- stat.close();
-
- sqlQueryBuilder = new StringBuilder("CREATE TABLE newbooks (");
- sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
- sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
- sqlQueryBuilder.append("qty INT DEFAULT NULL,");
- sqlQueryBuilder.append("PRIMARY KEY (id))");
-
- stat = conn.createStatement();
- stat.executeUpdate(sqlQueryBuilder.toString());
- stat.close();
-
- sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
- sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
- sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
- sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
- sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
- sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
-
- stat = conn.createStatement();
- stat.execute(sqlQueryBuilder.toString());
- stat.close();
-
- conn.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
deleted file mode 100644
index b76f8b8..0000000
--- a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.io.jdbc;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.junit.Assert;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class JDBCInputFormatTest {
- JDBCInputFormat jdbcInputFormat;
-
- static Connection conn;
-
- static final Object[][] dbData = {
- {1001, ("Java for dummies"), ("Tan Ah Teck"), 11.11, 11},
- {1002, ("More Java for dummies"), ("Tan Ah Teck"), 22.22, 22},
- {1003, ("More Java for more dummies"), ("Mohammad Ali"), 33.33, 33},
- {1004, ("A Cup of Java"), ("Kumar"), 44.44, 44},
- {1005, ("A Teaspoon of Java"), ("Kevin Jones"), 55.55, 55}};
-
- @BeforeClass
- public static void setUpClass() {
- try {
- prepareDerbyDatabase();
- } catch (Exception e) {
- Assert.fail();
- }
- }
-
- private static void prepareDerbyDatabase() throws ClassNotFoundException, SQLException {
- System.setProperty("derby.stream.error.field", "org.apache.flink.api.java.record.io.jdbc.DevNullLogStream.DEV_NULL");
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
- conn = DriverManager.getConnection(dbURL);
- createTable();
- insertDataToSQLTable();
- conn.close();
- }
-
- private static void createTable() throws SQLException {
- StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE books (");
- sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
- sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
- sqlQueryBuilder.append("qty INT DEFAULT NULL,");
- sqlQueryBuilder.append("PRIMARY KEY (id))");
-
- Statement stat = conn.createStatement();
- stat.executeUpdate(sqlQueryBuilder.toString());
- stat.close();
- }
-
- private static void insertDataToSQLTable() throws SQLException {
- StringBuilder sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
- sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
- sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
- sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
- sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
- sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
-
- Statement stat = conn.createStatement();
- stat.execute(sqlQueryBuilder.toString());
- stat.close();
- }
-
- @AfterClass
- public static void tearDownClass() {
- cleanUpDerbyDatabases();
- }
-
- private static void cleanUpDerbyDatabases() {
- try {
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
-
- conn = DriverManager.getConnection(dbURL);
- Statement stat = conn.createStatement();
- stat.executeUpdate("DROP TABLE books");
- stat.close();
- conn.close();
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- @After
- public void tearDown() {
- jdbcInputFormat = null;
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidDriver() throws IOException {
- jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
- .setDrivername("org.apache.derby.jdbc.idontexist")
- .setDBUrl("jdbc:derby:memory:ebookshop")
- .setQuery("select * from books")
- .finish();
- jdbcInputFormat.open(null);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidURL() throws IOException {
- jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setDBUrl("jdbc:der:iamanerror:mory:ebookshop")
- .setQuery("select * from books")
- .finish();
- jdbcInputFormat.open(null);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidQuery() throws IOException {
- jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setDBUrl("jdbc:derby:memory:ebookshop")
- .setQuery("iamnotsql")
- .finish();
- jdbcInputFormat.open(null);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testIncompleteConfiguration() throws IOException {
- jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setQuery("select * from books")
- .finish();
- }
-
- @Test(expected = IOException.class)
- public void testIncompatibleTuple() throws IOException {
- jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setDBUrl("jdbc:derby:memory:ebookshop")
- .setQuery("select * from books")
- .finish();
- jdbcInputFormat.open(null);
- jdbcInputFormat.nextRecord(new Tuple2());
- }
-
- @Test
- public void testJDBCInputFormat() throws IOException {
- jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setDBUrl("jdbc:derby:memory:ebookshop")
- .setQuery("select * from books")
- .finish();
- jdbcInputFormat.open(null);
- Tuple5 tuple = new Tuple5();
- int recordCount = 0;
- while (!jdbcInputFormat.reachedEnd()) {
- jdbcInputFormat.nextRecord(tuple);
- Assert.assertEquals("Field 0 should be int", Integer.class, tuple.getField(0).getClass());
- Assert.assertEquals("Field 1 should be String", String.class, tuple.getField(1).getClass());
- Assert.assertEquals("Field 2 should be String", String.class, tuple.getField(2).getClass());
- Assert.assertEquals("Field 3 should be float", Double.class, tuple.getField(3).getClass());
- Assert.assertEquals("Field 4 should be int", Integer.class, tuple.getField(4).getClass());
-
- for (int x = 0; x < 5; x++) {
- Assert.assertEquals(dbData[recordCount][x], tuple.getField(x));
- }
- recordCount++;
- }
- Assert.assertEquals(5, recordCount);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
deleted file mode 100644
index 7d004f9..0000000
--- a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.io.jdbc;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.junit.Assert;
-
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class JDBCOutputFormatTest {
- private JDBCInputFormat jdbcInputFormat;
- private JDBCOutputFormat jdbcOutputFormat;
-
- private static Connection conn;
-
- static final Object[][] dbData = {
- {1001, ("Java for dummies"), ("Tan Ah Teck"), 11.11, 11},
- {1002, ("More Java for dummies"), ("Tan Ah Teck"), 22.22, 22},
- {1003, ("More Java for more dummies"), ("Mohammad Ali"), 33.33, 33},
- {1004, ("A Cup of Java"), ("Kumar"), 44.44, 44},
- {1005, ("A Teaspoon of Java"), ("Kevin Jones"), 55.55, 55}};
-
- @BeforeClass
- public static void setUpClass() throws SQLException {
- try {
- System.setProperty("derby.stream.error.field", "org.apache.flink.api.java.record.io.jdbc.DevNullLogStream.DEV_NULL");
- prepareDerbyDatabase();
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- private static void prepareDerbyDatabase() throws ClassNotFoundException, SQLException {
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
- conn = DriverManager.getConnection(dbURL);
- createTable("books");
- createTable("newbooks");
- insertDataToSQLTables();
- conn.close();
- }
-
- private static void createTable(String tableName) throws SQLException {
- StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE ");
- sqlQueryBuilder.append(tableName);
- sqlQueryBuilder.append(" (");
- sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
- sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
- sqlQueryBuilder.append("qty INT DEFAULT NULL,");
- sqlQueryBuilder.append("PRIMARY KEY (id))");
-
- Statement stat = conn.createStatement();
- stat.executeUpdate(sqlQueryBuilder.toString());
- stat.close();
- }
-
- private static void insertDataToSQLTables() throws SQLException {
- StringBuilder sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
- sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
- sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
- sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
- sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
- sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
-
- Statement stat = conn.createStatement();
- stat.execute(sqlQueryBuilder.toString());
- stat.close();
- }
-
- @AfterClass
- public static void tearDownClass() {
- cleanUpDerbyDatabases();
- }
-
- private static void cleanUpDerbyDatabases() {
- try {
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
-
- conn = DriverManager.getConnection(dbURL);
- Statement stat = conn.createStatement();
- stat.executeUpdate("DROP TABLE books");
- stat.executeUpdate("DROP TABLE newbooks");
- stat.close();
- conn.close();
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- @After
- public void tearDown() {
- jdbcOutputFormat = null;
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidDriver() throws IOException {
- jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
- .setDrivername("org.apache.derby.jdbc.idontexist")
- .setDBUrl("jdbc:derby:memory:ebookshop")
- .setQuery("insert into books (id, title, author, price, qty) values (?,?,?,?,?)")
- .finish();
- jdbcOutputFormat.open(0, 1);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidURL() throws IOException {
- jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setDBUrl("jdbc:der:iamanerror:mory:ebookshop")
- .setQuery("insert into books (id, title, author, price, qty) values (?,?,?,?,?)")
- .finish();
- jdbcOutputFormat.open(0, 1);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidQuery() throws IOException {
- jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setDBUrl("jdbc:derby:memory:ebookshop")
- .setQuery("iamnotsql")
- .finish();
- jdbcOutputFormat.open(0, 1);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testIncompleteConfiguration() throws IOException {
- jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setQuery("insert into books (id, title, author, price, qty) values (?,?,?,?,?)")
- .finish();
- }
-
-
- @Test(expected = IllegalArgumentException.class)
- public void testIncompatibleTypes() throws IOException {
- jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
- .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
- .setDBUrl("jdbc:derby:memory:ebookshop")
- .setQuery("insert into books (id, title, author, price, qty) values (?,?,?,?,?)")
- .finish();
- jdbcOutputFormat.open(0, 1);
-
- Tuple5 tuple5 = new Tuple5();
- tuple5.setField(4, 0);
- tuple5.setField("hello", 1);
- tuple5.setField("world", 2);
- tuple5.setField(0.99, 3);
- tuple5.setField("imthewrongtype", 4);
-
- jdbcOutputFormat.writeRecord(tuple5);
- jdbcOutputFormat.close();
- }
-
- @Test
- public void testJDBCOutputFormat() throws IOException {
- String sourceTable = "books";
- String targetTable = "newbooks";
- String driverPath = "org.apache.derby.jdbc.EmbeddedDriver";
- String dbUrl = "jdbc:derby:memory:ebookshop";
-
- jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
- .setDBUrl(dbUrl)
- .setDrivername(driverPath)
- .setQuery("insert into " + targetTable + " (id, title, author, price, qty) values (?,?,?,?,?)")
- .finish();
- jdbcOutputFormat.open(0, 1);
-
- jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
- .setDrivername(driverPath)
- .setDBUrl(dbUrl)
- .setQuery("select * from " + sourceTable)
- .finish();
- jdbcInputFormat.open(null);
-
- Tuple5 tuple = new Tuple5();
- while (!jdbcInputFormat.reachedEnd()) {
- jdbcInputFormat.nextRecord(tuple);
- jdbcOutputFormat.writeRecord(tuple);
- }
-
- jdbcOutputFormat.close();
- jdbcInputFormat.close();
-
- jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
- .setDrivername(driverPath)
- .setDBUrl(dbUrl)
- .setQuery("select * from " + targetTable)
- .finish();
- jdbcInputFormat.open(null);
-
- int recordCount = 0;
- while (!jdbcInputFormat.reachedEnd()) {
- jdbcInputFormat.nextRecord(tuple);
- Assert.assertEquals("Field 0 should be int", Integer.class, tuple.getField(0).getClass());
- Assert.assertEquals("Field 1 should be String", String.class, tuple.getField(1).getClass());
- Assert.assertEquals("Field 2 should be String", String.class, tuple.getField(2).getClass());
- Assert.assertEquals("Field 3 should be float", Double.class, tuple.getField(3).getClass());
- Assert.assertEquals("Field 4 should be int", Integer.class, tuple.getField(4).getClass());
-
- for (int x = 0; x < 5; x++) {
- Assert.assertEquals(dbData[recordCount][x], tuple.getField(x));
- }
-
- recordCount++;
- }
- Assert.assertEquals(5, recordCount);
-
- jdbcInputFormat.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/DevNullLogStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/DevNullLogStream.java b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/DevNullLogStream.java
deleted file mode 100644
index 172f585..0000000
--- a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/DevNullLogStream.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.java.record.io.jdbc;
-
-import java.io.OutputStream;
-
-public class DevNullLogStream {
-
- public static final OutputStream DEV_NULL = new OutputStream() {
- public void write(int b) {}
- };
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java
deleted file mode 100644
index 8e0a2c5..0000000
--- a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.java.record.io.jdbc;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.junit.Assert;
-
-import org.apache.flink.api.java.record.io.jdbc.JDBCInputFormat;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.types.DoubleValue;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.StringValue;
-import org.apache.flink.types.Value;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class JDBCInputFormatTest {
- JDBCInputFormat jdbcInputFormat;
- Configuration config;
- static Connection conn;
- static final Value[][] dbData = {
- {new IntValue(1001), new StringValue("Java for dummies"), new StringValue("Tan Ah Teck"), new DoubleValue(11.11), new IntValue(11)},
- {new IntValue(1002), new StringValue("More Java for dummies"), new StringValue("Tan Ah Teck"), new DoubleValue(22.22), new IntValue(22)},
- {new IntValue(1003), new StringValue("More Java for more dummies"), new StringValue("Mohammad Ali"), new DoubleValue(33.33), new IntValue(33)},
- {new IntValue(1004), new StringValue("A Cup of Java"), new StringValue("Kumar"), new DoubleValue(44.44), new IntValue(44)},
- {new IntValue(1005), new StringValue("A Teaspoon of Java"), new StringValue("Kevin Jones"), new DoubleValue(55.55), new IntValue(55)}};
-
- @BeforeClass
- public static void setUpClass() {
- try {
- prepareDerbyDatabase();
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- private static void prepareDerbyDatabase() throws ClassNotFoundException {
- System.setProperty("derby.stream.error.field","org.apache.flink.api.java.record.io.jdbc.DevNullLogStream.DEV_NULL");
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- createConnection(dbURL);
- }
-
- private static void cleanUpDerbyDatabases() {
- try {
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
- conn = DriverManager.getConnection(dbURL);
- Statement stat = conn.createStatement();
- stat.executeUpdate("DROP TABLE books");
- stat.close();
- conn.close();
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- /*
- Loads JDBC derby driver ; creates(if necessary) and populates database.
- */
- private static void createConnection(String dbURL) {
- try {
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
- conn = DriverManager.getConnection(dbURL);
- createTable();
- insertDataToSQLTables();
- conn.close();
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- private static void createTable() throws SQLException {
- StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE books (");
- sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
- sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
- sqlQueryBuilder.append("qty INT DEFAULT NULL,");
- sqlQueryBuilder.append("PRIMARY KEY (id))");
-
- Statement stat = conn.createStatement();
- stat.executeUpdate(sqlQueryBuilder.toString());
- stat.close();
-
- sqlQueryBuilder = new StringBuilder("CREATE TABLE bookscontent (");
- sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
- sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("content BLOB(10K) DEFAULT NULL,");
- sqlQueryBuilder.append("PRIMARY KEY (id))");
-
- stat = conn.createStatement();
- stat.executeUpdate(sqlQueryBuilder.toString());
- stat.close();
- }
-
- private static void insertDataToSQLTables() throws SQLException {
- StringBuilder sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
- sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
- sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
- sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
- sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
- sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
-
- Statement stat = conn.createStatement();
- stat.execute(sqlQueryBuilder.toString());
- stat.close();
-
- sqlQueryBuilder = new StringBuilder("INSERT INTO bookscontent (id, title, content) VALUES ");
- sqlQueryBuilder.append("(1001, 'Java for dummies', CAST(X'7f454c4602' AS BLOB)),");
- sqlQueryBuilder.append("(1002, 'More Java for dummies', CAST(X'7f454c4602' AS BLOB)),");
- sqlQueryBuilder.append("(1003, 'More Java for more dummies', CAST(X'7f454c4602' AS BLOB)),");
- sqlQueryBuilder.append("(1004, 'A Cup of Java', CAST(X'7f454c4602' AS BLOB)),");
- sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', CAST(X'7f454c4602' AS BLOB))");
-
- stat = conn.createStatement();
- stat.execute(sqlQueryBuilder.toString());
- stat.close();
- }
-
-
- @After
- public void tearDown() {
- jdbcInputFormat = null;
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidConnection() {
- jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:idontexist", "select * from books;");
- jdbcInputFormat.configure(null);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidQuery() {
- jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:ebookshop", "abc");
- jdbcInputFormat.configure(null);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidDBType() {
- jdbcInputFormat = new JDBCInputFormat("idontexist.Driver", "jdbc:derby:memory:ebookshop", "select * from books;");
- jdbcInputFormat.configure(null);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testUnsupportedSQLType() {
- jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:ebookshop", "select * from bookscontent");
- jdbcInputFormat.configure(null);
- jdbcInputFormat.nextRecord(new Record());
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testNotConfiguredFormatNext() {
- jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:ebookshop", "select * from books");
- jdbcInputFormat.nextRecord(new Record());
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testNotConfiguredFormatEnd() {
- jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:ebookshop", "select * from books");
- jdbcInputFormat.reachedEnd();
- }
-
- @Test
- public void testJDBCInputFormat() throws IOException {
- jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:ebookshop", "select * from books");
- jdbcInputFormat.configure(null);
- Record record = new Record();
- int recordCount = 0;
- while (!jdbcInputFormat.reachedEnd()) {
- jdbcInputFormat.nextRecord(record);
- Assert.assertEquals(5, record.getNumFields());
- Assert.assertEquals("Field 0 should be int", IntValue.class, record.getField(0, IntValue.class).getClass());
- Assert.assertEquals("Field 1 should be String", StringValue.class, record.getField(1, StringValue.class).getClass());
- Assert.assertEquals("Field 2 should be String", StringValue.class, record.getField(2, StringValue.class).getClass());
- Assert.assertEquals("Field 3 should be float", DoubleValue.class, record.getField(3, DoubleValue.class).getClass());
- Assert.assertEquals("Field 4 should be int", IntValue.class, record.getField(4, IntValue.class).getClass());
-
- int[] pos = {0, 1, 2, 3, 4};
- Value[] values = {new IntValue(), new StringValue(), new StringValue(), new DoubleValue(), new IntValue()};
- Assert.assertTrue(record.equalsFields(pos, dbData[recordCount], values));
-
- recordCount++;
- }
- Assert.assertEquals(5, recordCount);
-
- cleanUpDerbyDatabases();
- }
-
-}
[27/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
deleted file mode 100644
index 177a9ee..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
+++ /dev/null
@@ -1,573 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala
-
-import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream,
- SingleOutputStreamOperator, GroupedDataStream}
-import scala.reflect.ClassTag
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.common.functions.MapFunction
-import org.apache.flink.streaming.api.invokable.operator.MapInvokable
-import org.apache.flink.util.Collector
-import org.apache.flink.api.common.functions.FlatMapFunction
-import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable
-import org.apache.flink.api.common.functions.ReduceFunction
-import org.apache.flink.streaming.api.invokable.StreamInvokable
-import org.apache.flink.streaming.api.invokable.operator.{ GroupedReduceInvokable, StreamReduceInvokable }
-import org.apache.flink.api.common.functions.ReduceFunction
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.api.common.functions.FilterFunction
-import org.apache.flink.streaming.api.function.sink.SinkFunction
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
-import org.apache.flink.streaming.api.windowing.policy.{ EvictionPolicy, TriggerPolicy }
-import org.apache.flink.streaming.api.collector.OutputSelector
-import scala.collection.JavaConversions._
-import java.util.HashMap
-import org.apache.flink.streaming.api.function.aggregation.SumFunction
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
-import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy
-
-class DataStream[T](javaStream: JavaStream[T]) {
-
- /**
- * Gets the underlying java DataStream object.
- */
- def getJavaStream: JavaStream[T] = javaStream
-
- /**
- * Sets the degree of parallelism of this operation. This must be greater than 1.
- */
- def setParallelism(dop: Int): DataStream[T] = {
- javaStream match {
- case ds: SingleOutputStreamOperator[_, _] => ds.setParallelism(dop)
- case _ =>
- throw new UnsupportedOperationException("Operator " + javaStream.toString + " cannot " +
- "have " +
- "parallelism.")
- }
- this
- }
-
- /**
- * Returns the degree of parallelism of this operation.
- */
- def getParallelism: Int = javaStream match {
- case op: SingleOutputStreamOperator[_, _] => op.getParallelism
- case _ =>
- throw new UnsupportedOperationException("Operator " + javaStream.toString + " does not have" +
- " " +
- "parallelism.")
- }
-
- def setChainingStrategy(strategy: ChainingStrategy): DataStream[T] = {
- javaStream match {
- case ds: SingleOutputStreamOperator[_, _] => ds.setChainingStrategy(strategy)
- case _ =>
- throw new UnsupportedOperationException("Only supported for operators.")
- }
- this
- }
-
- /**
- * Creates a new DataStream by merging DataStream outputs of
- * the same type with each other. The DataStreams merged using this operator
- * will be transformed simultaneously.
- *
- */
- def merge(dataStreams: DataStream[T]*): DataStream[T] =
- javaStream.merge(dataStreams.map(_.getJavaStream): _*)
-
- /**
- * Creates a new ConnectedDataStream by connecting
- * DataStream outputs of different type with each other. The
- * DataStreams connected using this operators can be used with CoFunctions.
- *
- */
- def connect[T2](dataStream: DataStream[T2]): ConnectedDataStream[T, T2] =
- javaStream.connect(dataStream.getJavaStream)
-
- /**
- * Groups the elements of a DataStream by the given key positions (for tuple/array types) to
- * be used with grouped operators like grouped reduce or grouped aggregations
- *
- */
- def groupBy(fields: Int*): DataStream[T] = javaStream.groupBy(fields: _*)
-
- /**
- * Groups the elements of a DataStream by the given field expressions to
- * be used with grouped operators like grouped reduce or grouped aggregations
- *
- */
- def groupBy(firstField: String, otherFields: String*): DataStream[T] =
- javaStream.groupBy(firstField +: otherFields.toArray: _*)
-
- /**
- * Groups the elements of a DataStream by the given K key to
- * be used with grouped operators like grouped reduce or grouped aggregations
- *
- */
- def groupBy[K: TypeInformation](fun: T => K): DataStream[T] = {
-
- val keyExtractor = new KeySelector[T, K] {
- val cleanFun = clean(fun)
- def getKey(in: T) = cleanFun(in)
- }
- javaStream.groupBy(keyExtractor)
- }
-
- /**
- * Sets the partitioning of the DataStream so that the output tuples
- * are broadcasted to every parallel instance of the next component. This
- * setting only effects the how the outputs will be distributed between the
- * parallel instances of the next processing operator.
- *
- */
- def broadcast: DataStream[T] = javaStream.broadcast()
-
- /**
- * Sets the partitioning of the DataStream so that the output values all go to
- * the first instance of the next processing operator. Use this setting with care
- * since it might cause a serious performance bottleneck in the application.
- */
- def global: DataStream[T] = javaStream.global()
-
- /**
- * Sets the partitioning of the DataStream so that the output tuples
- * are shuffled to the next component. This setting only effects the how the
- * outputs will be distributed between the parallel instances of the next
- * processing operator.
- *
- */
- def shuffle: DataStream[T] = javaStream.shuffle()
-
- /**
- * Sets the partitioning of the DataStream so that the output tuples
- * are forwarded to the local subtask of the next component (whenever
- * possible). This is the default partitioner setting. This setting only
- * effects the how the outputs will be distributed between the parallel
- * instances of the next processing operator.
- *
- */
- def forward: DataStream[T] = javaStream.forward()
-
- /**
- * Sets the partitioning of the DataStream so that the output tuples
- * are distributed evenly to the next component.This setting only effects
- * the how the outputs will be distributed between the parallel instances of
- * the next processing operator.
- *
- */
- def distribute: DataStream[T] = javaStream.distribute()
-
- /**
- * Initiates an iterative part of the program that creates a loop by feeding
- * back data streams. To create a streaming iteration the user needs to define
- * a transformation that creates two DataStreams.The first one one is the output
- * that will be fed back to the start of the iteration and the second is the output
- * stream of the iterative part.
- * <p>
- * stepfunction: initialStream => (feedback, output)
- * <p>
- * A common pattern is to use output splitting to create feedback and output DataStream.
- * Please refer to the .split(...) method of the DataStream
- * <p>
- * By default a DataStream with iteration will never terminate, but the user
- * can use the maxWaitTime parameter to set a max waiting time for the iteration head.
- * If no data received in the set time the stream terminates.
- *
- *
- */
- def iterate[R](stepFunction: DataStream[T] => (DataStream[T], DataStream[R]),
- maxWaitTimeMillis:Long = 0): DataStream[R] = {
- val iterativeStream = javaStream.iterate(maxWaitTimeMillis)
-
- val (feedback, output) = stepFunction(new DataStream[T](iterativeStream))
- iterativeStream.closeWith(feedback.getJavaStream)
- output
- }
-
- /**
- * Applies an aggregation that that gives the current maximum of the data stream at
- * the given position.
- *
- */
- def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
-
- /**
- * Applies an aggregation that that gives the current maximum of the data stream at
- * the given field.
- *
- */
- def max(field: String): DataStream[T] = aggregate(AggregationType.MAX, field)
-
- /**
- * Applies an aggregation that that gives the current minimum of the data stream at
- * the given position.
- *
- */
- def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
-
- /**
- * Applies an aggregation that that gives the current minimum of the data stream at
- * the given field.
- *
- */
- def min(field: String): DataStream[T] = aggregate(AggregationType.MIN, field)
-
- /**
- * Applies an aggregation that sums the data stream at the given position.
- *
- */
- def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
-
- /**
- * Applies an aggregation that sums the data stream at the given field.
- *
- */
- def sum(field: String): DataStream[T] = aggregate(AggregationType.SUM, field)
-
- /**
- * Applies an aggregation that that gives the current minimum element of the data stream by
- * the given position. When equality, the first element is returned with the minimal value.
- *
- */
- def minBy(position: Int): DataStream[T] = aggregate(AggregationType
- .MINBY, position)
-
- /**
- * Applies an aggregation that that gives the current minimum element of the data stream by
- * the given field. When equality, the first element is returned with the minimal value.
- *
- */
- def minBy(field: String): DataStream[T] = aggregate(AggregationType
- .MINBY, field )
-
- /**
- * Applies an aggregation that that gives the current maximum element of the data stream by
- * the given position. When equality, the first element is returned with the maximal value.
- *
- */
- def maxBy(position: Int): DataStream[T] =
- aggregate(AggregationType.MAXBY, position)
-
- /**
- * Applies an aggregation that that gives the current maximum element of the data stream by
- * the given field. When equality, the first element is returned with the maximal value.
- *
- */
- def maxBy(field: String): DataStream[T] =
- aggregate(AggregationType.MAXBY, field)
-
- private def aggregate(aggregationType: AggregationType, field: String): DataStream[T] = {
- val position = fieldNames2Indices(javaStream.getType(), Array(field))(0)
- aggregate(aggregationType, position)
- }
-
- private def aggregate(aggregationType: AggregationType, position: Int):
- DataStream[T] = {
-
- val jStream = javaStream.asInstanceOf[JavaStream[Product]]
- val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
-
- val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
-
- val reducer = aggregationType match {
- case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(outType.getTypeAt(position).
- getTypeClass()))
- case _ => new agg.ProductComparableAggregator(aggregationType, true)
- }
-
- val invokable = jStream match {
- case groupedStream: GroupedDataStream[_] => new GroupedReduceInvokable(reducer,
- groupedStream.getKeySelector())
- case _ => new StreamReduceInvokable(reducer)
- }
- new DataStream[Product](jStream.transform("aggregation", jStream.getType(),
- invokable)).asInstanceOf[DataStream[T]]
- }
-
- /**
- * Creates a new DataStream containing the current number (count) of
- * received records.
- *
- */
- def count: DataStream[Long] = new DataStream[java.lang.Long](
- javaStream.count()).asInstanceOf[DataStream[Long]]
-
- /**
- * Creates a new DataStream by applying the given function to every element of this DataStream.
- */
- def map[R: TypeInformation: ClassTag](fun: T => R): DataStream[R] = {
- if (fun == null) {
- throw new NullPointerException("Map function must not be null.")
- }
- val mapper = new MapFunction[T, R] {
- val cleanFun = clean(fun)
- def map(in: T): R = cleanFun(in)
- }
-
- javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper))
- }
-
- /**
- * Creates a new DataStream by applying the given function to every element of this DataStream.
- */
- def map[R: TypeInformation: ClassTag](mapper: MapFunction[T, R]): DataStream[R] = {
- if (mapper == null) {
- throw new NullPointerException("Map function must not be null.")
- }
-
- javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper))
- }
-
- /**
- * Creates a new DataStream by applying the given function to every element and flattening
- * the results.
- */
- def flatMap[R: TypeInformation: ClassTag](flatMapper: FlatMapFunction[T, R]): DataStream[R] = {
- if (flatMapper == null) {
- throw new NullPointerException("FlatMap function must not be null.")
- }
- javaStream.transform("flatMap", implicitly[TypeInformation[R]],
- new FlatMapInvokable[T, R](flatMapper))
- }
-
- /**
- * Creates a new DataStream by applying the given function to every element and flattening
- * the results.
- */
- def flatMap[R: TypeInformation: ClassTag](fun: (T, Collector[R]) => Unit): DataStream[R] = {
- if (fun == null) {
- throw new NullPointerException("FlatMap function must not be null.")
- }
- val flatMapper = new FlatMapFunction[T, R] {
- val cleanFun = clean(fun)
- def flatMap(in: T, out: Collector[R]) { cleanFun(in, out) }
- }
- flatMap(flatMapper)
- }
-
- /**
- * Creates a new DataStream by applying the given function to every element and flattening
- * the results.
- */
- def flatMap[R: TypeInformation: ClassTag](fun: T => TraversableOnce[R]): DataStream[R] = {
- if (fun == null) {
- throw new NullPointerException("FlatMap function must not be null.")
- }
- val flatMapper = new FlatMapFunction[T, R] {
- val cleanFun = clean(fun)
- def flatMap(in: T, out: Collector[R]) { cleanFun(in) foreach out.collect }
- }
- flatMap(flatMapper)
- }
-
- /**
- * Creates a new [[DataStream]] by reducing the elements of this DataStream
- * using an associative reduce function.
- */
- def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
- if (reducer == null) {
- throw new NullPointerException("Reduce function must not be null.")
- }
- javaStream match {
- case ds: GroupedDataStream[_] => javaStream.transform("reduce",
- javaStream.getType(), new GroupedReduceInvokable[T](reducer, ds.getKeySelector()))
- case _ => javaStream.transform("reduce", javaStream.getType(),
- new StreamReduceInvokable[T](reducer))
- }
- }
-
- /**
- * Creates a new [[DataStream]] by reducing the elements of this DataStream
- * using an associative reduce function.
- */
- def reduce(fun: (T, T) => T): DataStream[T] = {
- if (fun == null) {
- throw new NullPointerException("Reduce function must not be null.")
- }
- val reducer = new ReduceFunction[T] {
- val cleanFun = clean(fun)
- def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
- }
- reduce(reducer)
- }
-
- /**
- * Creates a new DataStream that contains only the elements satisfying the given filter predicate.
- */
- def filter(filter: FilterFunction[T]): DataStream[T] = {
- if (filter == null) {
- throw new NullPointerException("Filter function must not be null.")
- }
- javaStream.filter(filter)
- }
-
- /**
- * Creates a new DataStream that contains only the elements satisfying the given filter predicate.
- */
- def filter(fun: T => Boolean): DataStream[T] = {
- if (fun == null) {
- throw new NullPointerException("Filter function must not be null.")
- }
- val filter = new FilterFunction[T] {
- val cleanFun = clean(fun)
- def filter(in: T) = cleanFun(in)
- }
- this.filter(filter)
- }
-
- /**
- * Create a WindowedDataStream that can be used to apply
- * transformation like .reduce(...) or aggregations on
- * preset chunks(windows) of the data stream. To define the windows one or
- * more WindowingHelper-s such as Time, Count and
- * Delta can be used.</br></br> When applied to a grouped data
- * stream, the windows (evictions) and slide sizes (triggers) will be
- * computed on a per group basis. </br></br> For more advanced control over
- * the trigger and eviction policies please use to
- * window(List(triggers), List(evicters))
- */
- def window(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
- javaStream.window(windowingHelper: _*)
-
- /**
- * Create a WindowedDataStream using the given TriggerPolicy-s and EvictionPolicy-s.
- * Windowing can be used to apply transformation like .reduce(...) or aggregations on
- * preset chunks(windows) of the data stream.</br></br>For most common
- * use-cases please refer to window(WindowingHelper[_]*)
- *
- */
- def window(triggers: List[TriggerPolicy[T]], evicters: List[EvictionPolicy[T]]):
- WindowedDataStream[T] = javaStream.window(triggers, evicters)
-
- /**
- *
- * Operator used for directing tuples to specific named outputs using an
- * OutputSelector. Calling this method on an operator creates a new
- * SplitDataStream.
- */
- def split(selector: OutputSelector[T]): SplitDataStream[T] = javaStream.split(selector)
-
- /**
- * Creates a new SplitDataStream that contains only the elements satisfying the
- * given output selector predicate.
- */
- def split(fun: T => String): SplitDataStream[T] = {
- if (fun == null) {
- throw new NullPointerException("OutputSelector must not be null.")
- }
- val selector = new OutputSelector[T] {
- val cleanFun = clean(fun)
- def select(in: T): java.lang.Iterable[String] = {
- List(cleanFun(in))
- }
- }
- split(selector)
- }
-
- /**
- * Initiates a temporal Join transformation that joins the elements of two
- * data streams on key equality over a specified time window.
- *
- * This method returns a StreamJoinOperator on which the
- * .onWindow(..) should be called to define the
- * window, and then the .where(..) and .equalTo(..) methods can be used to defin
- * the join keys.</p> The user can also use the apply method of the returned JoinedStream
- * to use custom join function.
- *
- */
- def join[R](stream: DataStream[R]): StreamJoinOperator[T, R] =
- new StreamJoinOperator[T, R](javaStream, stream.getJavaStream)
-
- /**
- * Initiates a temporal cross transformation that builds all pair
- * combinations of elements of both DataStreams, i.e., it builds a Cartesian
- * product.
- *
- * This method returns a StreamJoinOperator on which the
- * .onWindow(..) should be called to define the
- * window, and then the .where(..) and .equalTo(..) methods can be used to defin
- * the join keys.</p> The user can also use the apply method of the returned JoinedStream
- * to use custom join function.
- *
- */
- def cross[R](stream: DataStream[R]): StreamCrossOperator[T, R] =
- new StreamCrossOperator[T, R](javaStream, stream.getJavaStream)
-
- /**
- * Writes a DataStream to the standard output stream (stdout). For each
- * element of the DataStream the result of .toString is
- * written.
- *
- */
- def print(): DataStream[T] = javaStream.print()
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically, in every millis milliseconds. For
- * every element of the DataStream the result of .toString
- * is written.
- *
- */
- def writeAsText(path: String, millis: Long = 0): DataStream[T] =
- javaStream.writeAsText(path, millis)
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically, in every millis milliseconds. For
- * every element of the DataStream the result of .toString
- * is written.
- *
- */
- def writeAsCsv(path: String, millis: Long = 0): DataStream[T] =
- javaStream.writeAsCsv(path, millis)
-
- /**
- * Adds the given sink to this DataStream. Only streams with sinks added
- * will be executed once the StreamExecutionEnvironment.execute(...)
- * method is called.
- *
- */
- def addSink(sinkFuntion: SinkFunction[T]): DataStream[T] =
- javaStream.addSink(sinkFuntion)
-
- /**
- * Adds the given sink to this DataStream. Only streams with sinks added
- * will be executed once the StreamExecutionEnvironment.execute(...)
- * method is called.
- *
- */
- def addSink(fun: T => Unit): DataStream[T] = {
- if (fun == null) {
- throw new NullPointerException("Sink function must not be null.")
- }
- val sinkFunction = new SinkFunction[T] {
- val cleanFun = clean(fun)
- def invoke(in: T) = cleanFun(in)
- }
- this.addSink(sinkFunction)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
deleted file mode 100644
index 9e33f80..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala
-
-import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
-
-/**
- * The SplitDataStream represents an operator that has been split using an
- * {@link OutputSelector}. Named outputs can be selected using the
- * {@link #select} function. To apply a transformation on the whole output simply call
- * the appropriate method on this stream.
- *
- * @param <OUT>
- * The type of the output.
- */
-class SplitDataStream[T](javaStream: SplitJavaStream[T]) extends DataStream[T](javaStream){
-
- /**
- * Sets the output names for which the next operator will receive values.
- */
- def select(outputNames: String*): DataStream[T] = javaStream.select(outputNames: _*)
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
deleted file mode 100644
index a408ec0..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala
-
-import scala.reflect.ClassTag
-import org.apache.commons.lang.Validate
-import org.apache.flink.api.common.functions.CrossFunction
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.api.scala.typeutils.CaseClassSerializer
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream}
-import org.apache.flink.streaming.api.function.co.CrossWindowFunction
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
-import java.util.concurrent.TimeUnit
-
-class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
- TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
-
- override def createNextWindowOperator(): StreamCrossOperator.CrossWindow[I1, I2] = {
-
- val crossWindowFunction = StreamCrossOperator.getCrossWindowFunction(this,
- (l: I1, r: I2) => (l, r))
-
- val returnType = new CaseClassTypeInfo[(I1, I2)](
-
- classOf[(I1, I2)], Seq(input1.getType, input2.getType), Array("_1", "_2")) {
-
- override def createSerializer: TypeSerializer[(I1, I2)] = {
- val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](getArity)
- for (i <- 0 until getArity) {
- fieldSerializers(i) = types(i).createSerializer
- }
-
- new CaseClassSerializer[(I1, I2)](classOf[(I1, I2)], fieldSerializers) {
- override def createInstance(fields: Array[AnyRef]) = {
- (fields(0).asInstanceOf[I1], fields(1).asInstanceOf[I2])
- }
- }
- }
- }
-
- val javaStream = input1.connect(input2).addGeneralWindowCombine(
- crossWindowFunction,
- returnType, windowSize,
- slideInterval, timeStamp1, timeStamp2)
-
- new StreamCrossOperator.CrossWindow[I1, I2](this, javaStream)
- }
-}
-object StreamCrossOperator {
-
- private[flink] class CrossWindow[I1, I2](op: StreamCrossOperator[I1, I2],
- javaStream: JavaStream[(I1, I2)]) extends
- DataStream[(I1, I2)](javaStream) with TemporalWindow[CrossWindow[I1, I2]] {
-
- /**
- * Sets a wrapper for the crossed elements. For each crossed pair, the result of the udf
- * call will be emitted.
- *
- */
- def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
-
- val invokable = new CoWindowInvokable[I1, I2, R](
- clean(getCrossWindowFunction(op, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
- op.timeStamp2)
-
- javaStream.getExecutionEnvironment().getStreamGraph().setInvokable(javaStream.getId(),
- invokable)
-
- javaStream.setType(implicitly[TypeInformation[R]])
- }
-
- override def every(length: Long, timeUnit: TimeUnit): CrossWindow[I1, I2] = {
- every(timeUnit.toMillis(length))
- }
-
- override def every(length: Long): CrossWindow[I1, I2] = {
- val builder = javaStream.getExecutionEnvironment().getStreamGraph()
- val invokable = builder.getInvokable(javaStream.getId())
- invokable.asInstanceOf[CoWindowInvokable[_,_,_]].setSlideSize(length)
- this
- }
- }
-
- private[flink] def getCrossWindowFunction[I1, I2, R](op: StreamCrossOperator[I1, I2],
- crossFunction: (I1, I2) => R):
- CrossWindowFunction[I1, I2, R] = {
- Validate.notNull(crossFunction, "Join function must not be null.")
-
- val crossFun = new CrossFunction[I1, I2, R] {
- val cleanFun = op.input1.clean(crossFunction)
-
- override def cross(first: I1, second: I2): R = {
- cleanFun(first, second)
- }
- }
-
- new CrossWindowFunction[I1, I2, R](crossFun)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
deleted file mode 100644
index 394673c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala
-
-import scala.reflect.ClassTag
-import org.apache.commons.lang.Validate
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JavaEnv}
-import org.apache.flink.streaming.api.function.source.{ FromElementsFunction, SourceFunction }
-import org.apache.flink.util.Collector
-import org.apache.flink.api.scala.ClosureCleaner
-import org.apache.flink.streaming.api.function.source.FileMonitoringFunction.WatchType
-
-class StreamExecutionEnvironment(javaEnv: JavaEnv) {
-
- /**
- * Sets the degree of parallelism (DOP) for operations executed through this environment.
- * Setting a DOP of x here will cause all operators (such as join, map, reduce) to run with
- * x parallel instances. This value can be overridden by specific operations using
- * [[DataStream.setParallelism]].
- */
- def setDegreeOfParallelism(degreeOfParallelism: Int): Unit = {
- javaEnv.setDegreeOfParallelism(degreeOfParallelism)
- }
-
- /**
- * Returns the default degree of parallelism for this execution environment. Note that this
- * value can be overridden by individual operations using [[DataStream.setParallelism]]
- */
- def getDegreeOfParallelism = javaEnv.getDegreeOfParallelism
-
- /**
- * Sets the maximum time frequency (milliseconds) for the flushing of the
- * output buffers. By default the output buffers flush frequently to provide
- * low latency and to aid smooth developer experience. Setting the parameter
- * can result in three logical modes:
- *
- * <ul>
- * <li>
- * A positive integer triggers flushing periodically by that integer</li>
- * <li>
- * 0 triggers flushing after every record thus minimizing latency</li>
- * <li>
- * -1 triggers flushing only when the output buffer is full thus maximizing
- * throughput</li>
- * </ul>
- *
- */
- def setBufferTimeout(timeoutMillis: Long): StreamExecutionEnvironment = {
- javaEnv.setBufferTimeout(timeoutMillis)
- this
- }
-
- /**
- * Gets the default buffer timeout set for this environment
- */
- def getBufferTimout: Long = javaEnv.getBufferTimeout()
-
- /**
- * Creates a DataStream that represents the Strings produced by reading the
- * given file line wise. The file will be read with the system's default
- * character set.
- *
- */
- def readTextFile(filePath: String): DataStream[String] =
- javaEnv.readTextFile(filePath)
-
- /**
- * Creates a DataStream that contains the contents of file created while
- * system watches the given path. The file will be read with the system's
- * default character set. The user can check the monitoring interval in milliseconds,
- * and the way file modifications are handled. By default it checks for only new files
- * every 100 milliseconds.
- *
- */
- def readFileStream(StreamPath: String, intervalMillis: Long = 100, watchType: WatchType =
- WatchType.ONLY_NEW_FILES): DataStream[String] =
- javaEnv.readFileStream(StreamPath, intervalMillis, watchType)
-
- /**
- * Creates a new DataStream that contains the strings received infinitely
- * from socket. Received strings are decoded by the system's default
- * character set.
- *
- */
- def socketTextStream(hostname: String, port: Int, delimiter: Char): DataStream[String] =
- javaEnv.socketTextStream(hostname, port, delimiter)
-
- /**
- * Creates a new DataStream that contains the strings received infinitely
- * from socket. Received strings are decoded by the system's default
- * character set, uses '\n' as delimiter.
- *
- */
- def socketTextStream(hostname: String, port: Int): DataStream[String] =
- javaEnv.socketTextStream(hostname, port)
-
- /**
- * Creates a new DataStream that contains a sequence of numbers.
- *
- */
- def generateSequence(from: Long, to: Long): DataStream[Long] = {
- new DataStream[java.lang.Long](javaEnv.generateSequence(from, to)).
- asInstanceOf[DataStream[Long]]
- }
-
- /**
- * Creates a DataStream that contains the given elements. The elements must all be of the
- * same type and must be serializable.
- *
- * * Note that this operation will result in a non-parallel data source, i.e. a data source with
- * a degree of parallelism of one.
- */
- def fromElements[T: ClassTag: TypeInformation](data: T*): DataStream[T] = {
- val typeInfo = implicitly[TypeInformation[T]]
- fromCollection(data)(implicitly[ClassTag[T]], typeInfo)
- }
-
- /**
- * Creates a DataStream from the given non-empty [[Seq]]. The elements need to be serializable
- * because the framework may move the elements into the cluster if needed.
- *
- * Note that this operation will result in a non-parallel data source, i.e. a data source with
- * a degree of parallelism of one.
- */
- def fromCollection[T: ClassTag: TypeInformation](
- data: Seq[T]): DataStream[T] = {
- Validate.notNull(data, "Data must not be null.")
- val typeInfo = implicitly[TypeInformation[T]]
-
- val sourceFunction = new FromElementsFunction[T](scala.collection.JavaConversions
- .asJavaCollection(data))
-
- javaEnv.addSource(sourceFunction, typeInfo)
- }
-
- /**
- * Create a DataStream using a user defined source function for arbitrary
- * source functionality. By default sources have a parallelism of 1.
- * To enable parallel execution, the user defined source should implement
- * ParallelSourceFunction or extend RichParallelSourceFunction.
- * In these cases the resulting source will have the parallelism of the environment.
- * To change this afterwards call DataStreamSource.setParallelism(int)
- *
- */
- def addSource[T: ClassTag: TypeInformation](function: SourceFunction[T]): DataStream[T] = {
- Validate.notNull(function, "Function must not be null.")
- val cleanFun = StreamExecutionEnvironment.clean(function)
- val typeInfo = implicitly[TypeInformation[T]]
- javaEnv.addSource(cleanFun, typeInfo)
- }
-
- /**
- * Create a DataStream using a user defined source function for arbitrary
- * source functionality.
- *
- */
- def addSource[T: ClassTag: TypeInformation](function: Collector[T] => Unit): DataStream[T] = {
- Validate.notNull(function, "Function must not be null.")
- val sourceFunction = new SourceFunction[T] {
- val cleanFun = StreamExecutionEnvironment.clean(function)
- override def invoke(out: Collector[T]) {
- cleanFun(out)
- }
- }
- addSource(sourceFunction)
- }
-
- /**
- * Triggers the program execution. The environment will execute all parts of
- * the program that have resulted in a "sink" operation. Sink operations are
- * for example printing results or forwarding them to a message queue.
- * <p>
- * The program execution will be logged and displayed with a generated
- * default name.
- *
- */
- def execute() = javaEnv.execute()
-
- /**
- * Triggers the program execution. The environment will execute all parts of
- * the program that have resulted in a "sink" operation. Sink operations are
- * for example printing results or forwarding them to a message queue.
- * <p>
- * The program execution will be logged and displayed with the provided name
- *
- */
- def execute(jobName: String) = javaEnv.execute(jobName)
-
- /**
- * Creates the plan with which the system will execute the program, and
- * returns it as a String using a JSON representation of the execution data
- * flow graph. Note that this needs to be called, before the plan is
- * executed.
- *
- */
- def getExecutionPlan() = javaEnv.getStreamGraph.getStreamingPlanAsJSON
-
-}
-
-object StreamExecutionEnvironment {
-
- private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
- ClosureCleaner.clean(f, checkSerializable)
- f
- }
-
- /**
- * Creates an execution environment that represents the context in which the program is
- * currently executed. If the program is invoked standalone, this method returns a local
- * execution environment. If the program is invoked from within the command line client
- * to be submitted to a cluster, this method returns the execution environment of this cluster.
- */
- def getExecutionEnvironment: StreamExecutionEnvironment = {
- new StreamExecutionEnvironment(JavaEnv.getExecutionEnvironment)
- }
-
- /**
- * Creates a local execution environment. The local execution environment will run the program in
- * a multi-threaded fashion in the same JVM as the environment was created in. The default degree
- * of parallelism of the local environment is the number of hardware contexts (CPU cores/threads).
- */
- def createLocalEnvironment(
- degreeOfParallelism: Int = Runtime.getRuntime.availableProcessors()):
- StreamExecutionEnvironment = {
- new StreamExecutionEnvironment(JavaEnv.createLocalEnvironment(degreeOfParallelism))
- }
-
- /**
- * Creates a remote execution environment. The remote environment sends (parts of) the program to
- * a cluster for execution. Note that all file paths used in the program must be accessible from
- * the cluster. The execution will use the cluster's default degree of parallelism, unless the
- * parallelism is set explicitly via [[StreamExecutionEnvironment.setDegreeOfParallelism()]].
- *
- * @param host The host name or address of the master (JobManager),
- * where the program should be executed.
- * @param port The port of the master (JobManager), where the program should be executed.
- * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the
- * program uses
- * user-defined functions, user-defined input formats, or any libraries,
- * those must be
- * provided in the JAR files.
- */
- def createRemoteEnvironment(host: String, port: Int, jarFiles: String*):
- StreamExecutionEnvironment = {
- new StreamExecutionEnvironment(JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*))
- }
-
- /**
- * Creates a remote execution environment. The remote environment sends (parts of) the program
- * to a cluster for execution. Note that all file paths used in the program must be accessible
- * from the cluster. The execution will use the specified degree of parallelism.
- *
- * @param host The host name or address of the master (JobManager),
- * where the program should be executed.
- * @param port The port of the master (JobManager), where the program should be executed.
- * @param degreeOfParallelism The degree of parallelism to use during the execution.
- * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the
- * program uses
- * user-defined functions, user-defined input formats, or any libraries,
- * those must be
- * provided in the JAR files.
- */
- def createRemoteEnvironment(
- host: String,
- port: Int,
- degreeOfParallelism: Int,
- jarFiles: String*): StreamExecutionEnvironment = {
- val javaEnv = JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*)
- javaEnv.setDegreeOfParallelism(degreeOfParallelism)
- new StreamExecutionEnvironment(javaEnv)
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
deleted file mode 100644
index 1bd1bfb..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala
-
-import scala.Array.canBuildFrom
-import scala.reflect.ClassTag
-import org.apache.commons.lang.Validate
-import org.apache.flink.api.common.functions.JoinFunction
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.api.java.operators.Keys
-import org.apache.flink.api.scala.typeutils.CaseClassSerializer
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
-import org.apache.flink.streaming.api.function.co.JoinWindowFunction
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.util.keys.KeySelectorUtil
-import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
-import java.util.concurrent.TimeUnit
-
-class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
-TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
-
- override def createNextWindowOperator() = {
- new StreamJoinOperator.JoinWindow[I1, I2](this)
- }
-}
-
-object StreamJoinOperator {
-
- class JoinWindow[I1, I2](private[flink]op: StreamJoinOperator[I1, I2]) extends
- TemporalWindow[JoinWindow[I1, I2]] {
-
- private[flink] val type1 = op.input1.getType()
-
- /**
- * Continues a temporal Join transformation by defining
- * the fields in the first stream to be used as keys for the join.
- * The resulting incomplete join can be completed by JoinPredicate.equalTo()
- * to define the second key.
- */
- def where(fields: Int*) = {
- new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(fields.toArray, type1), type1))
- }
-
- /**
- * Continues a temporal Join transformation by defining
- * the fields in the first stream to be used as keys for the join.
- * The resulting incomplete join can be completed by JoinPredicate.equalTo()
- * to define the second key.
- */
- def where(firstField: String, otherFields: String*) =
- new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(firstField +: otherFields.toArray, type1), type1))
-
- /**
- * Continues a temporal Join transformation by defining
- * the keyselector function that will be used to extract keys from the first stream
- * for the join.
- * The resulting incomplete join can be completed by JoinPredicate.equalTo()
- * to define the second key.
- */
- def where[K: TypeInformation](fun: (I1) => K) = {
- val keyType = implicitly[TypeInformation[K]]
- val keyExtractor = new KeySelector[I1, K] {
- val cleanFun = op.input1.clean(fun)
- def getKey(in: I1) = cleanFun(in)
- }
- new JoinPredicate[I1, I2](op, keyExtractor)
- }
-
- override def every(length: Long, timeUnit: TimeUnit): JoinWindow[I1, I2] = {
- every(timeUnit.toMillis(length))
- }
-
- override def every(length: Long): JoinWindow[I1, I2] = {
- op.slideInterval = length
- this
- }
-
- }
-
- class JoinPredicate[I1, I2](private[flink] val op: StreamJoinOperator[I1, I2],
- private[flink] val keys1: KeySelector[I1, _]) {
- private[flink] var keys2: KeySelector[I2, _] = null
- private[flink] val type2 = op.input2.getType()
-
- /**
- * Creates a temporal join transformation by defining the second join key.
- * The returned transformation wrapes each joined element pair in a tuple2:
- * (first, second)
- * To define a custom wrapping, use JoinedStream.apply(...)
- */
- def equalTo(fields: Int*): JoinedStream[I1, I2] = {
- finish(KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(fields.toArray, type2), type2))
- }
-
- /**
- * Creates a temporal join transformation by defining the second join key.
- * The returned transformation wrapes each joined element pair in a tuple2:
- * (first, second)
- * To define a custom wrapping, use JoinedStream.apply(...)
- */
- def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] =
- finish(KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(firstField +: otherFields.toArray, type2), type2))
-
- /**
- * Creates a temporal join transformation by defining the second join key.
- * The returned transformation wrapes each joined element pair in a tuple2:
- * (first, second)
- * To define a custom wrapping, use JoinedStream.apply(...)
- */
- def equalTo[K: TypeInformation](fun: (I2) => K): JoinedStream[I1, I2] = {
- val keyType = implicitly[TypeInformation[K]]
- val keyExtractor = new KeySelector[I2, K] {
- val cleanFun = op.input1.clean(fun)
- def getKey(in: I2) = cleanFun(in)
- }
- finish(keyExtractor)
- }
-
- private def finish(keys2: KeySelector[I2, _]): JoinedStream[I1, I2] = {
- this.keys2 = keys2
- new JoinedStream[I1, I2](this, createJoinOperator())
- }
-
- private def createJoinOperator(): JavaStream[(I1, I2)] = {
-
- val returnType = new CaseClassTypeInfo[(I1, I2)](
-
- classOf[(I1, I2)], Seq(op.input1.getType, op.input2.getType), Array("_1", "_2")) {
-
- override def createSerializer: TypeSerializer[(I1, I2)] = {
- val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](getArity)
- for (i <- 0 until getArity) {
- fieldSerializers(i) = types(i).createSerializer
- }
-
- new CaseClassSerializer[(I1, I2)](classOf[(I1, I2)], fieldSerializers) {
- override def createInstance(fields: Array[AnyRef]) = {
- (fields(0).asInstanceOf[I1], fields(1).asInstanceOf[I2])
- }
- }
- }
- }
-
- return op.input1.groupBy(keys1).connect(op.input2.groupBy(keys2))
- .addGeneralWindowCombine(getJoinWindowFunction(this, (_, _)),
- returnType, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
- }
- }
-
- class JoinedStream[I1, I2](jp: JoinPredicate[I1, I2], javaStream: JavaStream[(I1, I2)]) extends
- DataStream[(I1, I2)](javaStream) {
-
- private val op = jp.op
-
- /**
- * Sets a wrapper for the joined elements. For each joined pair, the result of the
- * udf call will be emitted.
- */
- def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
-
- val invokable = new CoWindowInvokable[I1, I2, R](
- clean(getJoinWindowFunction(jp, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
- op.timeStamp2)
-
- javaStream.getExecutionEnvironment().getStreamGraph().setInvokable(javaStream.getId(),
- invokable)
-
- javaStream.setType(implicitly[TypeInformation[R]])
- }
- }
-
- private[flink] def getJoinWindowFunction[I1, I2, R](jp: JoinPredicate[I1, I2],
- joinFunction: (I1, I2) => R) = {
- Validate.notNull(joinFunction, "Join function must not be null.")
-
- val joinFun = new JoinFunction[I1, I2, R] {
-
- val cleanFun = jp.op.input1.clean(joinFunction)
-
- override def join(first: I1, second: I2): R = {
- cleanFun(first, second)
- }
- }
-
- new JoinWindowFunction[I1, I2, R](jp.keys1, jp.keys2, joinFun)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala
deleted file mode 100644
index fd3a4a9..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala
-
-import org.apache.flink.streaming.api.datastream.temporaloperator.{ TemporalOperator => JTempOp }
-import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
-import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
-import org.apache.flink.streaming.api.windowing.helper.Timestamp
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment._
-
-abstract class TemporalOperator[I1, I2, OP <: TemporalWindow[OP]](
- i1: JavaStream[I1], i2: JavaStream[I2]) extends JTempOp[I1, I2, OP](i1, i2) {
-
- def onWindow(length: Long, ts1: I1 => Long, ts2: I2 => Long, startTime: Long = 0): OP = {
- val timeStamp1 = getTS(ts1)
- val timeStamp2 = getTS(ts2)
- onWindow(length, timeStamp1, timeStamp2, startTime)
- }
-
- def getTS[R](ts: R => Long): Timestamp[R] = {
- new Timestamp[R] {
- val cleanFun = clean(ts, true)
- def getTimestamp(in: R) = cleanFun(in)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
deleted file mode 100644
index 5c734bf..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala
-
-import scala.Array.canBuildFrom
-import scala.collection.JavaConversions.iterableAsScalaIterable
-import scala.reflect.ClassTag
-
-import org.apache.flink.api.common.functions.GroupReduceFunction
-import org.apache.flink.api.common.functions.ReduceFunction
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
-import org.apache.flink.streaming.api.datastream.{WindowedDataStream => JavaWStream}
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
-import org.apache.flink.streaming.api.function.aggregation.SumFunction
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
-import org.apache.flink.streaming.api.windowing.helper._
-import org.apache.flink.util.Collector
-
-class WindowedDataStream[T](javaStream: JavaWStream[T]) {
-
- /**
- * Defines the slide size (trigger frequency) for the windowed data stream.
- * This controls how often the user defined function will be triggered on
- * the window.
- */
- def every(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
- javaStream.every(windowingHelper: _*)
-
- /**
- * Groups the elements of the WindowedDataStream using the given
- * field positions. The window sizes (evictions) and slide sizes
- * (triggers) will be calculated on the whole stream (in a central fashion),
- * but the user defined functions will be applied on a per group basis.
- * </br></br> To get windows and triggers on a per group basis apply the
- * DataStream.window(...) operator on an already grouped data stream.
- *
- */
- def groupBy(fields: Int*): WindowedDataStream[T] = javaStream.groupBy(fields: _*)
-
- /**
- * Groups the elements of the WindowedDataStream using the given
- * field expressions. The window sizes (evictions) and slide sizes
- * (triggers) will be calculated on the whole stream (in a central fashion),
- * but the user defined functions will be applied on a per group basis.
- * </br></br> To get windows and triggers on a per group basis apply the
- * DataStream.window(...) operator on an already grouped data stream.
- *
- */
- def groupBy(firstField: String, otherFields: String*): WindowedDataStream[T] =
- javaStream.groupBy(firstField +: otherFields.toArray: _*)
-
- /**
- * Groups the elements of the WindowedDataStream using the given
- * KeySelector function. The window sizes (evictions) and slide sizes
- * (triggers) will be calculated on the whole stream (in a central fashion),
- * but the user defined functions will be applied on a per group basis.
- * </br></br> To get windows and triggers on a per group basis apply the
- * DataStream.window(...) operator on an already grouped data stream.
- *
- */
- def groupBy[K: TypeInformation](fun: T => K): WindowedDataStream[T] = {
-
- val keyExtractor = new KeySelector[T, K] {
- val cleanFun = clean(fun)
- def getKey(in: T) = cleanFun(in)
- }
- javaStream.groupBy(keyExtractor)
- }
-
- /**
- * Applies a reduce transformation on the windowed data stream by reducing
- * the current window at every trigger.
- *
- */
- def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
- if (reducer == null) {
- throw new NullPointerException("Reduce function must not be null.")
- }
- javaStream.reduce(reducer)
- }
-
- /**
- * Applies a reduce transformation on the windowed data stream by reducing
- * the current window at every trigger.
- *
- */
- def reduce(fun: (T, T) => T): DataStream[T] = {
- if (fun == null) {
- throw new NullPointerException("Reduce function must not be null.")
- }
- val reducer = new ReduceFunction[T] {
- val cleanFun = clean(fun)
- def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
- }
- reduce(reducer)
- }
-
- /**
- * Applies a reduceGroup transformation on the windowed data stream by reducing
- * the current window at every trigger. In contrast with the simple binary reduce operator,
- * groupReduce exposes the whole window through the Iterable interface.
- * </br>
- * </br>
- * Whenever possible try to use reduce instead of groupReduce for increased efficiency
- */
- def reduceGroup[R: ClassTag: TypeInformation](reducer: GroupReduceFunction[T, R]):
- DataStream[R] = {
- if (reducer == null) {
- throw new NullPointerException("GroupReduce function must not be null.")
- }
- javaStream.reduceGroup(reducer, implicitly[TypeInformation[R]])
- }
-
- /**
- * Applies a reduceGroup transformation on the windowed data stream by reducing
- * the current window at every trigger. In contrast with the simple binary reduce operator,
- * groupReduce exposes the whole window through the Iterable interface.
- * </br>
- * </br>
- * Whenever possible try to use reduce instead of groupReduce for increased efficiency
- */
- def reduceGroup[R: ClassTag: TypeInformation](fun: (Iterable[T], Collector[R]) => Unit):
- DataStream[R] = {
- if (fun == null) {
- throw new NullPointerException("GroupReduce function must not be null.")
- }
- val reducer = new GroupReduceFunction[T, R] {
- val cleanFun = clean(fun)
- def reduce(in: java.lang.Iterable[T], out: Collector[R]) = { cleanFun(in, out) }
- }
- reduceGroup(reducer)
- }
-
- /**
- * Applies an aggregation that that gives the maximum of the elements in the window at
- * the given position.
- *
- */
- def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
-
- /**
- * Applies an aggregation that that gives the maximum of the elements in the window at
- * the given field.
- *
- */
- def max(field: String): DataStream[T] = aggregate(AggregationType.MAX, field)
-
- /**
- * Applies an aggregation that that gives the minimum of the elements in the window at
- * the given position.
- *
- */
- def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
-
- /**
- * Applies an aggregation that that gives the minimum of the elements in the window at
- * the given field.
- *
- */
- def min(field: String): DataStream[T] = aggregate(AggregationType.MIN, field)
-
- /**
- * Applies an aggregation that sums the elements in the window at the given position.
- *
- */
- def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
-
- /**
- * Applies an aggregation that sums the elements in the window at the given field.
- *
- */
- def sum(field: String): DataStream[T] = aggregate(AggregationType.SUM, field)
-
- /**
- * Applies an aggregation that that gives the maximum element of the window by
- * the given position. When equality, returns the first.
- *
- */
- def maxBy(position: Int): DataStream[T] = aggregate(AggregationType.MAXBY,
- position)
-
- /**
- * Applies an aggregation that that gives the maximum element of the window by
- * the given field. When equality, returns the first.
- *
- */
- def maxBy(field: String): DataStream[T] = aggregate(AggregationType.MAXBY,
- field)
-
- /**
- * Applies an aggregation that that gives the minimum element of the window by
- * the given position. When equality, returns the first.
- *
- */
- def minBy(position: Int): DataStream[T] = aggregate(AggregationType.MINBY,
- position)
-
- /**
- * Applies an aggregation that that gives the minimum element of the window by
- * the given field. When equality, returns the first.
- *
- */
- def minBy(field: String): DataStream[T] = aggregate(AggregationType.MINBY,
- field)
-
- private def aggregate(aggregationType: AggregationType, field: String): DataStream[T] = {
- val position = fieldNames2Indices(javaStream.getType(), Array(field))(0)
- aggregate(aggregationType, position)
- }
-
- def aggregate(aggregationType: AggregationType, position: Int):
- DataStream[T] = {
-
- val jStream = javaStream.asInstanceOf[JavaWStream[Product]]
- val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
-
- val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
-
- val reducer = aggregationType match {
- case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(
- outType.getTypeAt(position).getTypeClass()))
- case _ => new agg.ProductComparableAggregator(aggregationType, true)
- }
-
- new DataStream[Product](jStream.reduce(reducer)).asInstanceOf[DataStream[T]]
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
deleted file mode 100644
index 222eb6d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api
-
-import _root_.scala.reflect.ClassTag
-import language.experimental.macros
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala.typeutils.{CaseClassTypeInfo, TypeUtils}
-import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
-import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
-import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
-import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaConStream }
-
-package object scala {
- // We have this here so that we always have generated TypeInformationS when
- // using the Scala API
- implicit def createTypeInformation[T]: TypeInformation[T] = macro TypeUtils.createTypeInfo[T]
-
- implicit def javaToScalaStream[R](javaStream: JavaStream[R]): DataStream[R] =
- new DataStream[R](javaStream)
-
- implicit def javaToScalaWindowedStream[R](javaWStream: JavaWStream[R]): WindowedDataStream[R] =
- new WindowedDataStream[R](javaWStream)
-
- implicit def javaToScalaSplitStream[R](javaStream: SplitJavaStream[R]): SplitDataStream[R] =
- new SplitDataStream[R](javaStream)
-
- implicit def javaToScalaConnectedStream[IN1, IN2](javaStream: JavaConStream[IN1, IN2]):
- ConnectedDataStream[IN1, IN2] = new ConnectedDataStream[IN1, IN2](javaStream)
-
- implicit def seqToFlinkSource[T: ClassTag: TypeInformation](scalaSeq: Seq[T]) : DataStream[T] =
- StreamExecutionEnvironment.getExecutionEnvironment.fromCollection(scalaSeq)
-
-
- private[flink] def fieldNames2Indices(
- typeInfo: TypeInformation[_],
- fields: Array[String]): Array[Int] = {
- typeInfo match {
- case ti: CaseClassTypeInfo[_] =>
- val result = ti.getFieldIndices(fields)
-
- if (result.contains(-1)) {
- throw new IllegalArgumentException("Fields '" + fields.mkString(", ") +
- "' are not valid for '" + ti.toString + "'.")
- }
-
- result
-
- case _ =>
- throw new UnsupportedOperationException("Specifying fields by name is only" +
- "supported on Case Classes (for now).")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala
deleted file mode 100644
index eedee0e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala.windowing
-
-import org.apache.flink.streaming.api.windowing.helper.{ Delta => JavaDelta }
-import org.apache.commons.lang.Validate
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction
-
-object Delta {
-
- /**
- * Creates a delta helper representing a delta trigger or eviction policy.
- * </br></br> This policy calculates a delta between the data point which
- * triggered last and the currently arrived data point. It triggers if the
- * delta is higher than a specified threshold. </br></br> In case it gets
- * used for eviction, this policy starts from the first element of the
- * buffer and removes all elements from the buffer which have a higher delta
- * then the threshold. As soon as there is an element with a lower delta,
- * the eviction stops.
- */
- def of[T](threshold: Double, deltaFunction: (T, T) => Double, initVal: T): JavaDelta[T] = {
- Validate.notNull(deltaFunction, "Delta function must not be null")
- val df = new DeltaFunction[T] {
- val cleanFun = clean(deltaFunction)
- override def getDelta(first: T, second: T) = cleanFun(first, second)
- }
- JavaDelta.of(threshold, df, initVal)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
deleted file mode 100644
index 9a69369..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala.windowing
-
-import java.util.concurrent.TimeUnit
-import org.apache.flink.streaming.api.windowing.helper.{ Time => JavaTime }
-
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.windowing.helper.Timestamp
-import org.apache.commons.lang.Validate
-
-object Time {
-
- /**
- * Creates a helper representing a time trigger which triggers every given
- * length (slide size) or a time eviction which evicts all elements older
- * than length (window size) using System time.
- *
- */
- def of(windowSize: Long, timeUnit: TimeUnit): JavaTime[_] =
- JavaTime.of(windowSize, timeUnit)
-
- /**
- * Creates a helper representing a time trigger which triggers every given
- * length (slide size) or a time eviction which evicts all elements older
- * than length (window size) using a user defined timestamp extractor.
- *
- */
- def of[R](windowSize: Long, timestamp: R => Long, startTime: Long = 0): JavaTime[R] = {
- Validate.notNull(timestamp, "Timestamp must not be null.")
- val ts = new Timestamp[R] {
- val fun = clean(timestamp, true)
- override def getTimestamp(in: R) = fun(in)
- }
- JavaTime.of(windowSize, ts, startTime)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/pom.xml b/flink-addons/flink-streaming/pom.xml
deleted file mode 100644
index 386ef73..0000000
--- a/flink-addons/flink-streaming/pom.xml
+++ /dev/null
@@ -1,75 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-addons</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-streaming-parent</artifactId>
- <name>flink-streaming</name>
- <packaging>pom</packaging>
-
- <modules>
- <module>flink-streaming-core</module>
- <module>flink-streaming-scala</module>
- <module>flink-streaming-examples</module>
- <module>flink-streaming-connectors</module>
- </modules>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-compiler</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-tachyon/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-tachyon/pom.xml b/flink-addons/flink-tachyon/pom.xml
deleted file mode 100644
index de36546..0000000
--- a/flink-addons/flink-tachyon/pom.xml
+++ /dev/null
@@ -1,117 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-addons</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-tachyon</artifactId>
- <name>flink-tachyon</name>
-
- <packaging>jar</packaging>
-
- <!--
- This is a Hadoop2 only flink module.
- -->
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java-examples</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-avro</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.tachyonproject</groupId>
- <artifactId>tachyon</artifactId>
- <version>0.5.0</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.tachyonproject</groupId>
- <artifactId>tachyon</artifactId>
- <version>0.5.0</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- <version>7.6.8.v20121106</version><!--$NO-MVN-MAN-VER$-->
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- <version>${hadoop.version}</version><!--$NO-MVN-MAN-VER$-->
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- <version>${hadoop.version}</version><!--$NO-MVN-MAN-VER$-->
- </dependency>
- </dependencies>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- <version>7.6.8.v20121106</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- <version>7.6.8.v20121106</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/HDFSTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/HDFSTest.java b/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/HDFSTest.java
deleted file mode 100644
index 7318894..0000000
--- a/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/HDFSTest.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tachyon;
-
-
-import org.apache.commons.io.IOUtils;
-import org.apache.flink.api.common.io.FileOutputFormat;
-import org.apache.flink.api.java.io.AvroOutputFormat;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.core.fs.FSDataInputStream;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.examples.java.wordcount.WordCount;
-import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.StringWriter;
-
-/**
- * This test should logically be located in the 'flink-runtime' tests. However, this project
- * has already all dependencies required (flink-java-examples). Also, the DOPOneExecEnv is here.
- */
-public class HDFSTest {
-
- private String hdfsURI;
- private MiniDFSCluster hdfsCluster;
- private org.apache.hadoop.fs.Path hdPath;
- private org.apache.hadoop.fs.FileSystem hdfs;
-
- @Before
- public void createHDFS() {
- try {
- Configuration hdConf = new Configuration();
-
- File baseDir = new File("./target/hdfs/hdfsTest").getAbsoluteFile();
- FileUtil.fullyDelete(baseDir);
- hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
- MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
- hdfsCluster = builder.build();
-
- hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() +"/";
-
- hdPath = new org.apache.hadoop.fs.Path("/test");
- hdfs = hdPath.getFileSystem(hdConf);
- FSDataOutputStream stream = hdfs.create(hdPath);
- for(int i = 0; i < 10; i++) {
- stream.write("Hello HDFS\n".getBytes());
- }
- stream.close();
-
- } catch(Throwable e) {
- e.printStackTrace();
- Assert.fail("Test failed " + e.getMessage());
- }
- }
-
- @After
- public void destroyHDFS() {
- try {
- hdfs.delete(hdPath, false);
- hdfsCluster.shutdown();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- }
-
- @Test
- public void testHDFS() {
-
- Path file = new Path(hdfsURI + hdPath);
- org.apache.hadoop.fs.Path result = new org.apache.hadoop.fs.Path(hdfsURI + "/result");
- try {
- FileSystem fs = file.getFileSystem();
- Assert.assertTrue("Must be HadoopFileSystem", fs instanceof HadoopFileSystem);
- new TachyonFileSystemWrapperTest.DopOneTestEnvironment();
- try {
- WordCount.main(new String[]{file.toString(), result.toString()});
- } catch(Throwable t) {
- t.printStackTrace();
- Assert.fail("Test failed with " + t.getMessage());
- }
- Assert.assertTrue("No result file present", hdfs.exists(result));
- // validate output:
- org.apache.hadoop.fs.FSDataInputStream inStream = hdfs.open(result);
- StringWriter writer = new StringWriter();
- IOUtils.copy(inStream, writer);
- String resultString = writer.toString();
-
- Assert.assertEquals("hdfs 10\n" +
- "hello 10\n", resultString);
- inStream.close();
-
- } catch (IOException e) {
- e.printStackTrace();
- Assert.fail("Error in test: " + e.getMessage() );
- }
- }
-
- @Test
- public void testAvroOut() {
- String type = "one";
- AvroOutputFormat<String> avroOut =
- new AvroOutputFormat<String>( String.class );
-
- org.apache.hadoop.fs.Path result = new org.apache.hadoop.fs.Path(hdfsURI + "/avroTest");
-
- avroOut.setOutputFilePath(new Path(result.toString()));
- avroOut.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
- avroOut.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);
-
- try {
- avroOut.open(0, 2);
- avroOut.writeRecord(type);
- avroOut.close();
-
- avroOut.open(1, 2);
- avroOut.writeRecord(type);
- avroOut.close();
-
-
- Assert.assertTrue("No result file present", hdfs.exists(result));
- FileStatus[] files = hdfs.listStatus(result);
- Assert.assertEquals(2, files.length);
- for(FileStatus file : files) {
- Assert.assertTrue("1.avro".equals(file.getPath().getName()) || "2.avro".equals(file.getPath().getName()));
- }
-
- } catch (IOException e) {
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-}
[09/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java
new file mode 100644
index 0000000..c224ad4
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.io.Serializable;
+
+/**
+ * An eviction policy specifies under which condition data points should be
+ * deleted from the buffer. Deletions must be done only in the order the
+ * elements arrived. Therefore, the policy only returns the number of elements
+ * to evict on each element arrival.
+ *
+ * @param <DATA>
+ * the type of the data handled by this policy
+ */
+public interface EvictionPolicy<DATA> extends Serializable {
+
+ /**
+ * Proves if and how many elements should be deleted from the element
+ * buffer. The eviction takes place after the trigger and after the call to
+ * the UDF but before the adding of the new data point.
+ *
+ * @param datapoint
+ * data point the data point which arrived
+ * @param triggered
+ * Information whether the UDF was triggered or not
+ * @param bufferSize
+ * The current size of the element buffer at the invokable
+ * @return The number of elements to be deleted from the buffer
+ */
+ public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
new file mode 100644
index 0000000..ee8a3cb
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+
+/**
+ * This policy can be used to trigger and evict based on a punctuation which is
+ * present within the arriving data. Using this policy, one can react on an
+ * externally defined arbitrary windowing semantic.
+ *
+ * In case this policy is used for eviction, the complete buffer will get
+ * deleted in case the punctuation is detected.
+ *
+ * By default this policy does not react on fake elements. Wrap it in an
+ * {@link ActiveEvictionPolicyWrapper} to make it react on punctuation even in
+ * fake elements.
+ *
+ * @param <IN>
+ * The type of the input data handled by this policy. An
+ * {@link Extractor} can be used to extract DATA for IN.
+ * @param <DATA>
+ * The type of the punctuation. An {@link Extractor} can be used to
+ * extract DATA for IN.
+ */
+public class PunctuationPolicy<IN, DATA> implements CloneableTriggerPolicy<IN>,
+ CloneableEvictionPolicy<IN> {
+
+ /**
+ * auto generated version id
+ */
+ private static final long serialVersionUID = -8845130188912602498L;
+ private int counter = 0;
+ private Extractor<IN, DATA> extractor;
+ private DATA punctuation;
+
+ /**
+ * Creates the punctuation policy without using any extractor. To make this
+ * work IN and DATA must not be different types.
+ *
+ * @param punctuation
+ * the punctuation which leads to trigger/evict.
+ */
+ public PunctuationPolicy(DATA punctuation) {
+ this(punctuation, null);
+ }
+
+ /**
+ * Creates the punctuation policy which uses the specified extractor to
+ * isolate the punctuation from the data.
+ *
+ * @param punctuation
+ * the punctuation which leads to trigger/evict.
+ * @param extractor
+ * An {@link Extractor} which converts IN to DATA.
+ */
+ public PunctuationPolicy(DATA punctuation, Extractor<IN, DATA> extractor) {
+ this.punctuation = punctuation;
+ this.extractor = extractor;
+ }
+
+ @Override
+ public int notifyEviction(IN datapoint, boolean triggered, int bufferSize) {
+ if (notifyTrigger(datapoint)) {
+ int tmp = counter;
+ // As the current will be add after the eviction the counter needs
+ // to be set to one already
+ counter = 1;
+ return tmp;
+ } else {
+ counter++;
+ return 0;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public boolean notifyTrigger(IN datapoint) {
+ DATA tmp;
+
+ // eventually extract data
+ if (extractor == null) {
+ // unchecked convert (cannot check it here)
+ tmp = (DATA) datapoint;
+ } else {
+ tmp = extractor.extract(datapoint);
+ }
+
+ // compare data with punctuation
+ if (punctuation.equals(tmp)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public PunctuationPolicy<IN, DATA> clone() {
+ return new PunctuationPolicy<IN, DATA>(punctuation, extractor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
new file mode 100644
index 0000000..16c30fc
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.util.LinkedList;
+
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+
+/**
+ * This eviction policy evicts all elements which are older then a specified
+ * time. The time is measured using a given {@link Timestamp} implementation. A
+ * point in time is always represented as long. Therefore, the granularity can
+ * be set as long value as well.
+ *
+ * @param <DATA>
+ * The type of the incoming data points which are processed by this
+ * policy.
+ */
+public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
+ CloneableEvictionPolicy<DATA> {
+
+ /**
+ * auto generated version id
+ */
+ private static final long serialVersionUID = -1457476766124518220L;
+
+ private long granularity;
+ private TimestampWrapper<DATA> timestampWrapper;
+ private LinkedList<Long> buffer = new LinkedList<Long>();
+
+ /**
+ * This eviction policy evicts all elements which are older than a specified
+ * time. The time is measured using a given {@link Timestamp}
+ * implementation. A point in time is always represented as long. Therefore,
+ * the granularity can be set as long value as well. If this value is set to
+ * 2 the policy will evict all elements which are older as 2.
+ *
+ * <code>
+ * while (time(firstInBuffer)<current time-granularity){
+ * evict firstInBuffer;
+ * }
+ * </code>
+ *
+ * @param granularity
+ * The granularity of the eviction. If this value is set to 2 the
+ * policy will evict all elements which are older as 2(if
+ * (time(X)<current time-granularity) evict X).
+ * @param timestampWrapper
+ * The {@link TimestampWrapper} to measure the time with. This
+ * can be either user defined of provided by the API.
+ */
+ public TimeEvictionPolicy(long granularity, TimestampWrapper<DATA> timestampWrapper) {
+ this.timestampWrapper = timestampWrapper;
+ this.granularity = granularity;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public int notifyEvictionWithFakeElement(Object datapoint, int bufferSize) {
+ checkForDeleted(bufferSize);
+
+ long threshold;
+ try {
+ threshold = (Long) datapoint - granularity;
+ } catch (ClassCastException e) {
+ threshold = timestampWrapper.getTimestamp((DATA) datapoint) - granularity;
+ }
+
+ // return result
+ return deleteAndCountExpired(threshold);
+
+ }
+
+ @Override
+ public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
+
+ checkForDeleted(bufferSize);
+
+ // remember timestamp
+ long time = timestampWrapper.getTimestamp(datapoint);
+
+ // delete and count expired tuples
+ long threshold = time - granularity;
+ int counter = deleteAndCountExpired(threshold);
+
+ // Add current element to buffer
+ buffer.add(time);
+
+ // return result
+ return counter;
+
+ }
+
+ private void checkForDeleted(int bufferSize) {
+ // check for deleted tuples (deletes by other policies)
+ while (bufferSize < this.buffer.size()) {
+ this.buffer.removeFirst();
+ }
+ }
+
+ private int deleteAndCountExpired(long threshold) {
+ int counter = 0;
+ while (!buffer.isEmpty()) {
+
+ if (buffer.getFirst() <= threshold) {
+ buffer.removeFirst();
+ counter++;
+ } else {
+ break;
+ }
+ }
+ return counter;
+
+ }
+
+ @Override
+ public TimeEvictionPolicy<DATA> clone() {
+ return new TimeEvictionPolicy<DATA>(granularity, timestampWrapper);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
new file mode 100644
index 0000000..1e91b8e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.util.LinkedList;
+
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+
+/**
+ * This trigger policy triggers with regard to the time. The is measured using a
+ * given {@link Timestamp} implementation. A point in time is always represented
+ * as long. Therefore, parameters such as granularity and delay can be set as
+ * long value as well.
+ *
+ * @param <DATA>
+ * The type of the incoming data points which are processed by this
+ * policy.
+ */
+public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
+ CloneableTriggerPolicy<DATA> {
+
+ /**
+ * auto generated version id
+ */
+ private static final long serialVersionUID = -5122753802440196719L;
+
+ protected long startTime;
+ protected long granularity;
+ protected TimestampWrapper<DATA> timestampWrapper;
+ protected long delay;
+
+ /**
+ * This trigger policy triggers with regard to the time. The is measured
+ * using a given {@link Timestamp} implementation. A point in time is always
+ * represented as long. Therefore, parameters such as granularity can be set
+ * as long value as well. If this value for the granularity is set to 2 for
+ * example, the policy will trigger at every second point in time.
+ *
+ * @param granularity
+ * The granularity of the trigger. If this value is set to x the
+ * policy will trigger at every x-th time point
+ * @param timestampWrapper
+ * The {@link TimestampWrapper} to measure the time with. This
+ * can be either user defined of provided by the API.
+ * @param timeWrapper
+ * This policy creates fake elements to not miss windows in case
+ * no element arrived within the duration of the window. This
+ * extractor should wrap a long into such an element of type
+ * DATA.
+ */
+ public TimeTriggerPolicy(long granularity, TimestampWrapper<DATA> timestampWrapper) {
+ this(granularity, timestampWrapper, 0);
+ }
+
+ /**
+ * This is mostly the same as
+ * {@link TimeTriggerPolicy#TimeTriggerPolicy(long, Timestamp)}. In addition
+ * to granularity and timestamp a delay can be specified for the first
+ * trigger. If the start time given by the timestamp is x, the delay is y,
+ * and the granularity is z, the first trigger will happen at x+y+z.
+ *
+ * @param granularity
+ * The granularity of the trigger. If this value is set to 2 the
+ * policy will trigger at every second time point
+ * @param timestampWrapper
+ * The {@link TimestampWrapper} to measure the time with. This
+ * can be either user defined of provided by the API.
+ * @param delay
+ * A delay for the first trigger. If the start time given by the
+ * timestamp is x, the delay is y, and the granularity is z, the
+ * first trigger will happen at x+y+z.
+ * @param timeWrapper
+ * This policy creates fake elements to not miss windows in case
+ * no element arrived within the duration of the window. This
+ * extractor should wrap a long into such an element of type
+ * DATA.
+ */
+ public TimeTriggerPolicy(long granularity, TimestampWrapper<DATA> timestampWrapper, long delay) {
+ this.startTime = timestampWrapper.getStartTime() + delay;
+ this.timestampWrapper = timestampWrapper;
+ this.granularity = granularity;
+ this.delay = delay;
+ }
+
+ /**
+ * This method checks if we missed a window end. If this is the case we
+ * trigger the missed windows using fake elements.
+ */
+ @Override
+ public synchronized Object[] preNotifyTrigger(DATA datapoint) {
+ LinkedList<Object> fakeElements = new LinkedList<Object>();
+ // check if there is more then one window border missed
+ // use > here. In case >= would fit, the regular call will do the job.
+ while (timestampWrapper.getTimestamp(datapoint) >= startTime + granularity) {
+ startTime += granularity;
+ fakeElements.add(startTime - 1);
+ }
+ return (Object[]) fakeElements.toArray();
+ }
+
+ /**
+ * In case {@link DefaultTimeStamp} is used, a runnable is returned which
+ * triggers based on the current system time. If any other time measure is
+ * used the method return null.
+ *
+ * @param callback
+ * The object which is takes the callbacks for adding fake
+ * elements out of the runnable.
+ * @return A runnable is returned which triggers based on the current system
+ * time. If any other time measure is used the method return null.
+ */
+ @Override
+ public Runnable createActiveTriggerRunnable(ActiveTriggerCallback callback) {
+ if (this.timestampWrapper.isDefaultTimestamp()) {
+ return new TimeCheck(callback);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * This method is only called in case the runnable triggers a window end
+ * according to the {@link DefaultTimeStamp}.
+ *
+ * @param callback
+ * The callback object.
+ */
+ private synchronized void activeFakeElementEmission(ActiveTriggerCallback callback) {
+
+ // start time is excluded, but end time is included: >=
+ if (System.currentTimeMillis() >= startTime + granularity) {
+ startTime += granularity;
+ callback.sendFakeElement(startTime - 1);
+ }
+
+ }
+
+ private class TimeCheck implements Runnable {
+ ActiveTriggerCallback callback;
+
+ public TimeCheck(ActiveTriggerCallback callback) {
+ this.callback = callback;
+ }
+
+ @Override
+ public void run() {
+ while (true) {
+ // wait for the specified granularity
+ try {
+ Thread.sleep(granularity);
+ } catch (InterruptedException e) {
+ // ignore it...
+ }
+ // Trigger using the respective methods. Methods are
+ // synchronized to prevent race conditions between real and fake
+ // elements at the policy.
+ activeFakeElementEmission(callback);
+ }
+ }
+ }
+
+ @Override
+ public synchronized boolean notifyTrigger(DATA datapoint) {
+ long recordTime = timestampWrapper.getTimestamp(datapoint);
+ if (recordTime >= startTime + granularity) {
+ if (granularity != 0) {
+ startTime = recordTime - ((recordTime - startTime) % granularity);
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public TimeTriggerPolicy<DATA> clone() {
+ return new TimeTriggerPolicy<DATA>(granularity, timestampWrapper, delay);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java
new file mode 100644
index 0000000..c212df6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.io.Serializable;
+
+/**
+ * Proves and returns if a new window should be started. In case the trigger
+ * occurs (return value true) the UDF will be executed on the current element
+ * buffer without the last added element which is provided as parameter. This
+ * element will be added to the buffer after the execution of the UDF.
+ *
+ * @param <DATA>
+ * The data type which can be handled by this policy
+ */
+public interface TriggerPolicy<DATA> extends Serializable {
+
+ /**
+ * Proves and returns if a new window should be started. In case the trigger
+ * occurs (return value true) the UDF will be executed on the current
+ * element buffer without the last added element which is provided as
+ * parameter. This element will be added to the buffer after the execution
+ * of the UDF.
+ *
+ * There are possibly different strategies for eviction and triggering: 1)
+ * including last data point: Better/faster for count eviction 2) excluding
+ * last data point: Essentially required for time based eviction and delta
+ * rules As 2) is required for some policies and the benefit of using 1) is
+ * small for the others, policies are implemented according to 2).
+ *
+ * @param datapoint
+ * the data point which arrived
+ * @return true if the current windows should be closed, otherwise false. In
+ * true case the given data point will be part of the next window
+ * and will not be included in the current one.
+ */
+ public boolean notifyTrigger(DATA datapoint);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
new file mode 100644
index 0000000..9a2e3c9
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * This eviction policy deletes all elements from the buffer in case a trigger
+ * occurred. Therefore, it is the default eviction policy to be used for any
+ * tumbling window.
+ *
+ * By default this policy does not react on fake elements. Wrap it in an
+ * {@link ActiveEvictionPolicyWrapper} to make it clearing the buffer even on
+ * fake elements.
+ *
+ * @param <DATA>
+ * The type of the data points which is handled by this policy
+ */
+public class TumblingEvictionPolicy<DATA> implements CloneableEvictionPolicy<DATA> {
+
+ /**
+ * Auto generated version ID
+ */
+ private static final long serialVersionUID = -4018019069267281155L;
+
+ /**
+ * Counter for the current number of elements in the buffer
+ */
+ private int counter = 0;
+
+ /**
+ * This is the default constructor providing no special functionality. This
+ * eviction policy deletes all elements from the buffer in case a trigger
+ * occurred. Therefore, it is the default eviction policy to be used for any
+ * tumbling window.
+ */
+ public TumblingEvictionPolicy() {
+ // default constructor, no further logic needed
+ }
+
+ /**
+ * This constructor allows to set a custom start value for the element
+ * counter.
+ *
+ * This eviction policy deletes all elements from the buffer in case a
+ * trigger occurred. Therefore, it is the default eviction policy to be used
+ * for any tumbling window.
+ *
+ * @param startValue
+ * A start value for the element counter
+ */
+ public TumblingEvictionPolicy(int startValue) {
+ this.counter = startValue;
+ }
+
+ /**
+ * Deletes all elements from the buffer in case the trigger occurred.
+ */
+ @Override
+ public int notifyEviction(Object datapoint, boolean triggered, int bufferSize) {
+ if (triggered) {
+ // The current data point will be part of the next window!
+ // Therefore the counter needs to be set to one already.
+ int tmpCounter = counter;
+ counter = 1;
+ return tmpCounter;
+ } else {
+ counter++;
+ return 0;
+ }
+ }
+
+ @Override
+ public TumblingEvictionPolicy<DATA> clone() {
+ return new TumblingEvictionPolicy<DATA>(counter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/BlockingQueueBroker.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/BlockingQueueBroker.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/BlockingQueueBroker.java
new file mode 100755
index 0000000..3ee2508
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/BlockingQueueBroker.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.io;
+
+import java.util.concurrent.BlockingQueue;
+
+import org.apache.flink.runtime.iterative.concurrent.Broker;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+
+@SuppressWarnings("rawtypes")
+public class BlockingQueueBroker extends Broker<BlockingQueue<StreamRecord>> {
+ /**
+ * Singleton instance
+ */
+ private static final BlockingQueueBroker INSTANCE = new BlockingQueueBroker();
+
+ private BlockingQueueBroker() {
+ }
+
+ /**
+ * retrieve singleton instance
+ */
+ public static Broker<BlockingQueue<StreamRecord>> instance() {
+ return INSTANCE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoReaderIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoReaderIterator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoReaderIterator.java
new file mode 100755
index 0000000..ed90c03
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoReaderIterator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.io;
+
+import java.io.IOException;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.runtime.plugable.DeserializationDelegate;
+import org.apache.flink.runtime.plugable.ReusingDeserializationDelegate;
+
+/**
+ * A CoReaderIterator wraps a {@link CoRecordReader} producing records of two
+ * input types.
+ */
+public class CoReaderIterator<T1, T2> {
+
+ private final CoRecordReader<DeserializationDelegate<T1>, DeserializationDelegate<T2>> reader; // the
+ // source
+
+ protected final ReusingDeserializationDelegate<T1> delegate1;
+ protected final ReusingDeserializationDelegate<T2> delegate2;
+
+ public CoReaderIterator(
+ CoRecordReader<DeserializationDelegate<T1>, DeserializationDelegate<T2>> reader,
+ TypeSerializer<T1> serializer1, TypeSerializer<T2> serializer2) {
+ this.reader = reader;
+ this.delegate1 = new ReusingDeserializationDelegate<T1>(serializer1);
+ this.delegate2 = new ReusingDeserializationDelegate<T2>(serializer2);
+ }
+
+ public int next(T1 target1, T2 target2) throws IOException {
+ this.delegate1.setInstance(target1);
+ this.delegate2.setInstance(target2);
+
+ try {
+ return this.reader.getNextRecord(this.delegate1, this.delegate2);
+
+ } catch (InterruptedException e) {
+ throw new IOException("Reader interrupted.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoRecordReader.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoRecordReader.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoRecordReader.java
new file mode 100755
index 0000000..0b1b373
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoRecordReader.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.io;
+
+import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.runtime.event.task.TaskEvent;
+import org.apache.flink.runtime.io.network.api.reader.BufferReaderBase;
+import org.apache.flink.runtime.io.network.api.reader.MutableRecordReader;
+import org.apache.flink.runtime.io.network.api.reader.ReaderBase;
+import org.apache.flink.runtime.io.network.api.serialization.AdaptiveSpanningRecordDeserializer;
+import org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer;
+import org.apache.flink.runtime.io.network.buffer.Buffer;
+import org.apache.flink.runtime.util.event.EventListener;
+
+import java.io.IOException;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+/**
+ * A CoRecordReader wraps {@link MutableRecordReader}s of two different input
+ * types to read records effectively.
+ */
+@SuppressWarnings("rawtypes")
+public class CoRecordReader<T1 extends IOReadableWritable, T2 extends IOReadableWritable> implements ReaderBase, EventListener<BufferReaderBase> {
+
+ private final BufferReaderBase bufferReader1;
+
+ private final BufferReaderBase bufferReader2;
+
+ private final BlockingQueue<Integer> availableRecordReaders = new LinkedBlockingQueue<Integer>();
+
+ private AdaptiveSpanningRecordDeserializer[] reader1RecordDeserializers;
+
+ private RecordDeserializer<T1> reader1currentRecordDeserializer;
+
+ private AdaptiveSpanningRecordDeserializer[] reader2RecordDeserializers;
+
+ private RecordDeserializer<T2> reader2currentRecordDeserializer;
+
+ // 0 => none, 1 => reader (T1), 2 => reader (T2)
+ private int currentReaderIndex;
+
+ private boolean hasRequestedPartitions;
+
+ public CoRecordReader(BufferReaderBase bufferReader1, BufferReaderBase bufferReader2) {
+ this.bufferReader1 = bufferReader1;
+ this.bufferReader2 = bufferReader2;
+
+ this.reader1RecordDeserializers = new AdaptiveSpanningRecordDeserializer[bufferReader1.getNumberOfInputChannels()];
+ this.reader2RecordDeserializers = new AdaptiveSpanningRecordDeserializer[bufferReader2.getNumberOfInputChannels()];
+
+ for (int i = 0; i < reader1RecordDeserializers.length; i++) {
+ reader1RecordDeserializers[i] = new AdaptiveSpanningRecordDeserializer<T1>();
+ }
+
+ for (int i = 0; i < reader2RecordDeserializers.length; i++) {
+ reader2RecordDeserializers[i] = new AdaptiveSpanningRecordDeserializer<T2>();
+ }
+
+ bufferReader1.subscribeToReader(this);
+ bufferReader2.subscribeToReader(this);
+ }
+
+ public void requestPartitionsOnce() throws IOException {
+ if (!hasRequestedPartitions) {
+ bufferReader1.requestPartitionsOnce();
+ bufferReader2.requestPartitionsOnce();
+
+ hasRequestedPartitions = true;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ protected int getNextRecord(T1 target1, T2 target2) throws IOException, InterruptedException {
+
+ requestPartitionsOnce();
+
+ while (true) {
+ if (currentReaderIndex == 0) {
+ if ((bufferReader1.isFinished() && bufferReader2.isFinished())) {
+ return 0;
+ }
+
+ currentReaderIndex = getNextReaderIndexBlocking();
+ }
+
+ if (currentReaderIndex == 1) {
+ while (true) {
+ if (reader1currentRecordDeserializer != null) {
+ RecordDeserializer.DeserializationResult result = reader1currentRecordDeserializer.getNextRecord(target1);
+
+ if (result.isBufferConsumed()) {
+ reader1currentRecordDeserializer.getCurrentBuffer().recycle();
+ reader1currentRecordDeserializer = null;
+
+ currentReaderIndex = 0;
+ }
+
+ if (result.isFullRecord()) {
+ return 1;
+ }
+ }
+
+ final Buffer nextBuffer = bufferReader1.getNextBufferBlocking();
+ final int channelIndex = bufferReader1.getChannelIndexOfLastBuffer();
+
+ if (nextBuffer == null) {
+ currentReaderIndex = 0;
+
+ break;
+ }
+
+ reader1currentRecordDeserializer = reader1RecordDeserializers[channelIndex];
+ reader1currentRecordDeserializer.setNextBuffer(nextBuffer);
+ }
+ }
+ else if (currentReaderIndex == 2) {
+ while (true) {
+ if (reader2currentRecordDeserializer != null) {
+ RecordDeserializer.DeserializationResult result = reader2currentRecordDeserializer.getNextRecord(target2);
+
+ if (result.isBufferConsumed()) {
+ reader2currentRecordDeserializer.getCurrentBuffer().recycle();
+ reader2currentRecordDeserializer = null;
+
+ currentReaderIndex = 0;
+ }
+
+ if (result.isFullRecord()) {
+ return 2;
+ }
+ }
+
+ final Buffer nextBuffer = bufferReader2.getNextBufferBlocking();
+ final int channelIndex = bufferReader2.getChannelIndexOfLastBuffer();
+
+ if (nextBuffer == null) {
+ currentReaderIndex = 0;
+
+ break;
+ }
+
+ reader2currentRecordDeserializer = reader2RecordDeserializers[channelIndex];
+ reader2currentRecordDeserializer.setNextBuffer(nextBuffer);
+ }
+ }
+ else {
+ throw new IllegalStateException("Bug: unexpected current reader index.");
+ }
+ }
+ }
+
+ private int getNextReaderIndexBlocking() throws InterruptedException {
+ return availableRecordReaders.take();
+ }
+
+ // ------------------------------------------------------------------------
+ // Data availability notifications
+ // ------------------------------------------------------------------------
+
+ @Override
+ public void onEvent(BufferReaderBase bufferReader) {
+ if (bufferReader == bufferReader1) {
+ availableRecordReaders.add(1);
+ }
+ else if (bufferReader == bufferReader2) {
+ availableRecordReaders.add(2);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public boolean isFinished() {
+ return bufferReader1.isFinished() && bufferReader2.isFinished();
+ }
+
+ @Override
+ public void subscribeToTaskEvent(EventListener<TaskEvent> eventListener, Class<? extends TaskEvent> eventType) {
+ bufferReader1.subscribeToTaskEvent(eventListener, eventType);
+ bufferReader2.subscribeToTaskEvent(eventListener, eventType);
+ }
+
+ @Override
+ public void sendTaskEvent(TaskEvent event) throws IOException, InterruptedException {
+ bufferReader1.sendTaskEvent(event);
+ bufferReader2.sendTaskEvent(event);
+ }
+
+ @Override
+ public void setIterativeReader() {
+ bufferReader1.setIterativeReader();
+ bufferReader2.setIterativeReader();
+ }
+
+ @Override
+ public void startNextSuperstep() {
+ bufferReader1.startNextSuperstep();
+ bufferReader2.startNextSuperstep();
+ }
+
+ @Override
+ public boolean hasReachedEndOfSuperstep() {
+ return bufferReader1.hasReachedEndOfSuperstep() && bufferReader2.hasReachedEndOfSuperstep();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
new file mode 100755
index 0000000..b7af589
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.io;
+
+import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.runtime.io.network.api.writer.BufferWriter;
+import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+import org.apache.flink.runtime.io.network.api.writer.RoundRobinChannelSelector;
+
+import java.io.IOException;
+
+public class StreamRecordWriter<T extends IOReadableWritable> extends RecordWriter<T> {
+
+ private long timeout;
+
+ private OutputFlusher outputFlusher;
+
+ public StreamRecordWriter(BufferWriter writer) {
+ this(writer, new RoundRobinChannelSelector<T>(), 1000);
+ }
+
+ public StreamRecordWriter(BufferWriter writer, ChannelSelector<T> channelSelector) {
+ this(writer, channelSelector, 1000);
+ }
+
+ public StreamRecordWriter(BufferWriter writer, ChannelSelector<T> channelSelector, long timeout) {
+ super(writer, channelSelector);
+
+ this.timeout = timeout;
+ this.outputFlusher = new OutputFlusher();
+
+ outputFlusher.start();
+ }
+
+ public void close() {
+ try {
+ if (outputFlusher != null) {
+ outputFlusher.terminate();
+ outputFlusher.join();
+ }
+
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private class OutputFlusher extends Thread {
+ private boolean running = true;
+ public void terminate() {
+ running = false;
+ }
+ @Override
+ public void run() {
+ while (running && !writer.isFinished()) {
+ try {
+ flush();
+ Thread.sleep(timeout);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java
new file mode 100644
index 0000000..3813c8a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+
+/**
+ * Partitioner that selects all the output channels.
+ *
+ * @param <T>
+ * Type of the Tuple
+ */
+public class BroadcastPartitioner<T> extends StreamPartitioner<T> {
+ private static final long serialVersionUID = 1L;
+
+ int[] returnArray;
+ boolean set;
+ int setNumber;
+
+ public BroadcastPartitioner() {
+ super(PartitioningStrategy.BROADCAST);
+ }
+
+ @Override
+ public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
+ int numberOfOutputChannels) {
+ if (set && setNumber == numberOfOutputChannels) {
+ return returnArray;
+ } else {
+ this.returnArray = new int[numberOfOutputChannels];
+ for (int i = 0; i < numberOfOutputChannels; i++) {
+ returnArray[i] = i;
+ }
+ set = true;
+ setNumber = numberOfOutputChannels;
+ return returnArray;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java
new file mode 100644
index 0000000..7299397
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+
+/**
+ * Partitioner that distributes the data equally by cycling through the output
+ * channels.
+ *
+ * @param <T>
+ * Type of the Tuple
+ */
+public class DistributePartitioner<T> extends StreamPartitioner<T> {
+ private static final long serialVersionUID = 1L;
+
+ private int[] returnArray = new int[] {-1};
+
+ public DistributePartitioner(boolean forward) {
+ super(forward ? PartitioningStrategy.FORWARD : PartitioningStrategy.DISTRIBUTE);
+ }
+
+ @Override
+ public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
+ int numberOfOutputChannels) {
+ this.returnArray[0] = (this.returnArray[0] + 1) % numberOfOutputChannels;
+
+ return this.returnArray;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java
new file mode 100644
index 0000000..6bc036e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+
+/**
+ * Partitioner that selects the same (one) channel for two Tuples having a
+ * specified fields equal.
+ *
+ * @param <T>
+ * Type of the Tuple
+ */
+public class FieldsPartitioner<T> extends StreamPartitioner<T> {
+ private static final long serialVersionUID = 1L;
+
+ private int[] returnArray = new int[1];;
+ KeySelector<T, ?> keySelector;
+
+ public FieldsPartitioner(KeySelector<T, ?> keySelector) {
+ super(PartitioningStrategy.GROUPBY);
+ this.keySelector = keySelector;
+ }
+
+ @Override
+ public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
+ int numberOfOutputChannels) {
+ returnArray[0] = Math.abs(record.getInstance().getKey(keySelector).hashCode()
+ % numberOfOutputChannels);
+
+ return returnArray;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java
new file mode 100644
index 0000000..c73ca71
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+
+//Group to the partitioner with the lowest id
+public class GlobalPartitioner<T> extends StreamPartitioner<T> {
+ private static final long serialVersionUID = 1L;
+
+ private int[] returnArray = new int[] { 0 };
+
+ public GlobalPartitioner() {
+ super(PartitioningStrategy.GLOBAL);
+ }
+
+ @Override
+ public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
+ int numberOfOutputChannels) {
+ return returnArray;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java
new file mode 100644
index 0000000..318de3f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import java.util.Random;
+
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+
+/**
+ * Partitioner that distributes the data equally by selecting one output channel
+ * randomly.
+ *
+ * @param <T>
+ * Type of the Tuple
+ */
+public class ShufflePartitioner<T> extends StreamPartitioner<T> {
+ private static final long serialVersionUID = 1L;
+
+ private Random random = new Random();
+
+ private int[] returnArray = new int[1];
+
+ public ShufflePartitioner() {
+ super(PartitioningStrategy.SHUFFLE);
+ }
+
+ @Override
+ public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
+ int numberOfOutputChannels) {
+ returnArray[0] = random.nextInt(numberOfOutputChannels);
+ return returnArray;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java
new file mode 100644
index 0000000..19a8dba
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.partitioner;
+
+import java.io.Serializable;
+
+import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+
+public abstract class StreamPartitioner<T> implements
+ ChannelSelector<SerializationDelegate<StreamRecord<T>>>, Serializable {
+
+ public enum PartitioningStrategy {
+
+ FORWARD, DISTRIBUTE, SHUFFLE, BROADCAST, GLOBAL, GROUPBY;
+
+ }
+
+ private static final long serialVersionUID = 1L;
+ private PartitioningStrategy strategy;
+
+ public StreamPartitioner(PartitioningStrategy strategy) {
+ this.strategy = strategy;
+ }
+
+ public PartitioningStrategy getStrategy() {
+ return strategy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/CircularFifoList.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/CircularFifoList.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/CircularFifoList.java
new file mode 100644
index 0000000..02a628b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/CircularFifoList.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+
+/**
+ * A simple class, that manages a circular queue with sliding interval. If the
+ * queue if full and a new element is added, the elements that belong to the
+ * first sliding interval are removed.
+ */
+public class CircularFifoList<T> implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private Queue<T> queue;
+ private Queue<Long> slideSizes;
+ private long counter;
+ private Iterable<T> iterable;
+
+ public CircularFifoList() {
+ this.queue = new LinkedList<T>();
+ this.slideSizes = new LinkedList<Long>();
+ this.counter = 0;
+ this.iterable = new ListIterable();
+ }
+
+ public void add(T element) {
+ queue.add(element);
+ counter++;
+ }
+
+ public void newSlide() {
+ slideSizes.add(counter);
+ counter = 0;
+ }
+
+ public void shiftWindow() {
+ shiftWindow(1);
+ }
+
+ public void shiftWindow(int numberOfSlides) {
+
+ if (numberOfSlides <= slideSizes.size()) {
+ for (int i = 0; i < numberOfSlides; i++) {
+ Long firstSlideSize = slideSizes.remove();
+
+ for (int j = 0; j < firstSlideSize; j++) {
+ queue.remove();
+ }
+ }
+ } else {
+ slideSizes.clear();
+ queue.clear();
+ counter = 0;
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<T> getElements(){
+ return (List<T>) queue;
+ }
+
+ public Iterator<T> getIterator() {
+ return queue.iterator();
+ }
+
+ public Iterable<T> getIterable() {
+ return iterable;
+ }
+
+ private class ListIterable implements Iterable<T>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Iterator<T> iterator() {
+ return getIterator();
+ }
+
+ }
+
+ public boolean isEmpty() {
+ return queue.isEmpty();
+ }
+
+ @Override
+ public String toString() {
+ return queue.toString();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/GraphState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/GraphState.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/GraphState.java
new file mode 100644
index 0000000..945c0fd
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/GraphState.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * GraphState represents a special {@link MapState} for storing graph
+ * structures.
+ *
+ */
+public class GraphState extends MapState<Integer, Set<Integer>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public GraphState() {
+ state = new HashMap<Integer, Set<Integer>>();
+ }
+
+ public void insertDirectedEdge(int sourceNode, int targetNode) {
+ if (!containsKey(sourceNode)) {
+ state.put(sourceNode, new HashSet<Integer>());
+ }
+ state.get(sourceNode).add(targetNode);
+ updatedItems.add(sourceNode);
+ removedItems.remove(sourceNode);
+ }
+
+ public void insertUndirectedEdge(int sourceNode, int targetNode) {
+ if (!state.containsKey(sourceNode)) {
+ state.put(sourceNode, new HashSet<Integer>());
+ }
+ if (!state.containsKey(targetNode)) {
+ state.put(targetNode, new HashSet<Integer>());
+ }
+ state.get(sourceNode).add(targetNode);
+ state.get(targetNode).add(sourceNode);
+
+ updatedItems.add(sourceNode);
+ removedItems.remove(sourceNode);
+ updatedItems.add(targetNode);
+ removedItems.remove(targetNode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/MapState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/MapState.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/MapState.java
new file mode 100644
index 0000000..85aec52
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/MapState.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.streaming.state.checkpoint.MapCheckpoint;
+import org.apache.flink.streaming.state.checkpoint.StateCheckpoint;
+
+/**
+ * A Map that can be used as a partitionable operator state, for both fault
+ * tolerance and rebalancing in Flink programs. The default repartitioner for
+ * this operator repartitions by the hashcodes of the keys in the map. </br>
+ * </br> The MapState also allows for incremental (data efficient) checkpointing
+ * of the state. The entries in the map should only be modified by using the
+ * dedicated methods: {@link #put(Object, Object)},{@link #remove(Object)},
+ * {@link #putAll(Map)} and {@link #clear}. Directly modifying the entryset
+ * will cause errors when checkpointing.
+ *
+ * @param <K>
+ * The type of the keys.
+ * @param <V>
+ * The type of the values.
+ */
+public class MapState<K, V> extends PartitionableState<Map<K, V>> implements Map<K, V> {
+
+ private static final long serialVersionUID = 1L;
+
+ protected Set<K> removedItems;
+ protected Set<K> updatedItems;
+ protected boolean clear;
+
+ public MapState() {
+ this.state = new HashMap<K, V>();
+ this.updatedItems = new HashSet<K>();
+ this.removedItems = new HashSet<K>();
+ }
+
+ @Override
+ public StateCheckpoint<Map<K, V>> checkpoint() {
+ this.updatedItems.removeAll(removedItems);
+ StateCheckpoint<Map<K, V>> checkpoint = new MapCheckpoint<K, V>(this);
+ resetHistory();
+ return checkpoint;
+ }
+
+ @Override
+ public OperatorState<Map<K, V>> restore(StateCheckpoint<Map<K, V>> checkpoint) {
+ this.state = checkpoint.getCheckpointedState();
+ resetHistory();
+ return this;
+ }
+
+ @Override
+ public OperatorState<Map<K, V>>[] repartition(int numberOfPartitions) {
+ @SuppressWarnings("unchecked")
+ MapState<K, V>[] states = new MapState[numberOfPartitions];
+ for (int i = 0; i < numberOfPartitions; i++) {
+ states[i] = new MapState<K, V>();
+ }
+ for (Entry<K, V> entry : this.state.entrySet()) {
+ int partition = Math.abs(entry.getKey().hashCode() % numberOfPartitions);
+ states[partition].state.put(entry.getKey(), entry.getValue());
+ }
+ return states;
+ }
+
+ @Override
+ public OperatorState<Map<K, V>> reBuild(OperatorState<Map<K, V>>... parts) {
+ clear();
+ for (OperatorState<Map<K, V>> operatorState : parts) {
+ putAll(operatorState.state);
+ }
+ return this;
+ }
+
+ public void resetHistory() {
+ clear = false;
+ removedItems.clear();
+ updatedItems.clear();
+ }
+
+ @Override
+ public int size() {
+ return state.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return state.isEmpty();
+ }
+
+ @Override
+ public boolean containsKey(Object key) {
+ return state.containsKey(key);
+ }
+
+ @Override
+ public boolean containsValue(Object value) {
+ return state.containsValue(value);
+ }
+
+ @Override
+ public V get(Object key) {
+ return state.get(key);
+ }
+
+ @Override
+ public V put(K key, V value) {
+ updatedItems.add(key);
+ removedItems.remove(key);
+ return state.put(key, value);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public V remove(Object key) {
+ V removed = state.remove(key);
+ if (removed != null) {
+ removedItems.add((K) key);
+ updatedItems.remove((K) key);
+ }
+
+ return removed;
+ }
+
+ @Override
+ public void putAll(Map<? extends K, ? extends V> m) {
+ for (K key : m.keySet()) {
+ updatedItems.add(key);
+ removedItems.remove(key);
+ }
+ state.putAll(m);
+ }
+
+ @Override
+ public void clear() {
+ clear = true;
+ updatedItems.clear();
+ removedItems.clear();
+ state.clear();
+ }
+
+ @Override
+ public Set<K> keySet() {
+ return state.keySet();
+ }
+
+ @Override
+ public Collection<V> values() {
+ return state.values();
+ }
+
+ @Override
+ public Set<Entry<K, V>> entrySet() {
+ return state.entrySet();
+ }
+
+ @Override
+ public Map<K, V> getState() {
+ throw new RuntimeException(
+ "State object should be accessed using the Map interface provided by the MapState");
+ }
+
+ @Override
+ public OperatorState<Map<K, V>> setState(Map<K, V> state) {
+ throw new RuntimeException(
+ "State object should be accessed using the Map interface provided by the MapState");
+ }
+
+ public Set<K> getRemovedItems() {
+ return removedItems;
+ }
+
+ public Set<K> getUpdatedItems() {
+ return updatedItems;
+ }
+
+ public boolean isCleared() {
+ return clear;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/NullableCircularBuffer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/NullableCircularBuffer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/NullableCircularBuffer.java
new file mode 100644
index 0000000..aa2a1bc
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/NullableCircularBuffer.java
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.AbstractCollection;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.commons.collections.BoundedCollection;
+import org.apache.commons.collections.Buffer;
+import org.apache.commons.collections.BufferUnderflowException;
+
+@SuppressWarnings("rawtypes")
+public class NullableCircularBuffer extends AbstractCollection implements Buffer,
+ BoundedCollection, Serializable {
+
+ /** Serialization version */
+ private static final long serialVersionUID = 5603722811189451017L;
+
+ /** Underlying storage array */
+ private transient Object[] elements;
+
+ /** Array index of first (oldest) buffer element */
+ private transient int start = 0;
+
+ /**
+ * Index mod maxElements of the array position following the last buffer
+ * element. Buffer elements start at elements[start] and "wrap around"
+ * elements[maxElements-1], ending at elements[decrement(end)]. For example,
+ * elements = {c,a,b}, start=1, end=1 corresponds to the buffer [a,b,c].
+ */
+ private transient int end = 0;
+
+ /** Flag to indicate if the buffer is currently full. */
+ private transient boolean full = false;
+
+ /** Capacity of the buffer */
+ private final int maxElements;
+
+ /**
+ * Constructs a new <code>BoundedFifoBuffer</code> big enough to hold 32
+ * elements.
+ */
+ public NullableCircularBuffer() {
+ this(32);
+ }
+
+ /**
+ * Constructs a new <code>BoundedFifoBuffer</code> big enough to hold the
+ * specified number of elements.
+ *
+ * @param size
+ * the maximum number of elements for this fifo
+ * @throws IllegalArgumentException
+ * if the size is less than 1
+ */
+ public NullableCircularBuffer(int size) {
+ if (size <= 0) {
+ throw new IllegalArgumentException("The size must be greater than 0");
+ }
+ elements = new Object[size];
+ maxElements = elements.length;
+ }
+
+ /**
+ * Constructs a new <code>BoundedFifoBuffer</code> big enough to hold all of
+ * the elements in the specified collection. That collection's elements will
+ * also be added to the buffer.
+ *
+ * @param coll
+ * the collection whose elements to add, may not be null
+ * @throws NullPointerException
+ * if the collection is null
+ */
+ @SuppressWarnings("unchecked")
+ public NullableCircularBuffer(Collection coll) {
+ this(coll.size());
+ addAll(coll);
+ }
+
+ // -----------------------------------------------------------------------
+ /**
+ * Write the buffer out using a custom routine.
+ *
+ * @param out
+ * the output stream
+ * @throws IOException
+ */
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.defaultWriteObject();
+ out.writeInt(size());
+ for (Iterator it = iterator(); it.hasNext();) {
+ out.writeObject(it.next());
+ }
+ }
+
+ /**
+ * Read the buffer in using a custom routine.
+ *
+ * @param in
+ * the input stream
+ * @throws IOException
+ * @throws ClassNotFoundException
+ */
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ elements = new Object[maxElements];
+ int size = in.readInt();
+ for (int i = 0; i < size; i++) {
+ elements[i] = in.readObject();
+ }
+ start = 0;
+ full = (size == maxElements);
+ if (full) {
+ end = 0;
+ } else {
+ end = size;
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ /**
+ * Returns the number of elements stored in the buffer.
+ *
+ * @return this buffer's size
+ */
+ public int size() {
+ int size = 0;
+
+ if (end < start) {
+ size = maxElements - start + end;
+ } else if (end == start) {
+ size = (full ? maxElements : 0);
+ } else {
+ size = end - start;
+ }
+
+ return size;
+ }
+
+ /**
+ * Returns true if this buffer is empty; false otherwise.
+ *
+ * @return true if this buffer is empty
+ */
+ public boolean isEmpty() {
+ return size() == 0;
+ }
+
+ /**
+ * Returns true if this collection is full and no new elements can be added.
+ *
+ * @return <code>true</code> if the collection is full
+ */
+ public boolean isFull() {
+ return size() == maxElements;
+ }
+
+ /**
+ * Gets the maximum size of the collection (the bound).
+ *
+ * @return the maximum number of elements the collection can hold
+ */
+ public int maxSize() {
+ return maxElements;
+ }
+
+ /**
+ * Clears this buffer.
+ */
+ public void clear() {
+ full = false;
+ start = 0;
+ end = 0;
+ Arrays.fill(elements, null);
+ }
+
+ /**
+ * Adds the given element to this buffer.
+ *
+ * @param element
+ * the element to add
+ * @return true, always
+ */
+ public boolean add(Object element) {
+
+ if (isFull()) {
+ remove();
+ }
+
+ elements[end++] = element;
+
+ if (end >= maxElements) {
+ end = 0;
+ }
+
+ if (end == start) {
+ full = true;
+ }
+
+ return true;
+ }
+
+ /**
+ * Returns the least recently inserted element in this buffer.
+ *
+ * @return the least recently inserted element
+ * @throws BufferUnderflowException
+ * if the buffer is empty
+ */
+ public Object get() {
+ if (isEmpty()) {
+ throw new BufferUnderflowException("The buffer is already empty");
+ }
+
+ return elements[start];
+ }
+
+ /**
+ * Removes the least recently inserted element from this buffer.
+ *
+ * @return the least recently inserted element
+ * @throws BufferUnderflowException
+ * if the buffer is empty
+ */
+ public Object remove() {
+ if (isEmpty()) {
+ throw new BufferUnderflowException("The buffer is already empty");
+ }
+
+ Object element = elements[start];
+
+ elements[start++] = null;
+
+ if (start >= maxElements) {
+ start = 0;
+ }
+
+ full = false;
+
+ return element;
+ }
+
+ /**
+ * Increments the internal index.
+ *
+ * @param index
+ * the index to increment
+ * @return the updated index
+ */
+ private int increment(int index) {
+ index++;
+ if (index >= maxElements) {
+ index = 0;
+ }
+ return index;
+ }
+
+ /**
+ * Decrements the internal index.
+ *
+ * @param index
+ * the index to decrement
+ * @return the updated index
+ */
+ private int decrement(int index) {
+ index--;
+ if (index < 0) {
+ index = maxElements - 1;
+ }
+ return index;
+ }
+
+ /**
+ * Returns an iterator over this buffer's elements.
+ *
+ * @return an iterator over this buffer's elements
+ */
+ public Iterator iterator() {
+ return new Iterator() {
+
+ private int index = start;
+ private int lastReturnedIndex = -1;
+ private boolean isFirst = full;
+
+ public boolean hasNext() {
+ return isFirst || (index != end);
+
+ }
+
+ public Object next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ isFirst = false;
+ lastReturnedIndex = index;
+ index = increment(index);
+ return elements[lastReturnedIndex];
+ }
+
+ public void remove() {
+ if (lastReturnedIndex == -1) {
+ throw new IllegalStateException();
+ }
+
+ // First element can be removed quickly
+ if (lastReturnedIndex == start) {
+ NullableCircularBuffer.this.remove();
+ lastReturnedIndex = -1;
+ return;
+ }
+
+ int pos = lastReturnedIndex + 1;
+ if (start < lastReturnedIndex && pos < end) {
+ // shift in one part
+ System.arraycopy(elements, pos, elements, lastReturnedIndex, end - pos);
+ } else {
+ // Other elements require us to shift the subsequent
+ // elements
+ while (pos != end) {
+ if (pos >= maxElements) {
+ elements[pos - 1] = elements[0];
+ pos = 0;
+ } else {
+ elements[decrement(pos)] = elements[pos];
+ pos = increment(pos);
+ }
+ }
+ }
+
+ lastReturnedIndex = -1;
+ end = decrement(end);
+ elements[end] = null;
+ full = false;
+ index = decrement(index);
+ }
+
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/OperatorState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/OperatorState.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/OperatorState.java
new file mode 100644
index 0000000..a0cedba
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/OperatorState.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state;
+
+import java.io.Serializable;
+
+import org.apache.flink.streaming.state.checkpoint.StateCheckpoint;
+
+/**
+ * Abstract class for representing operator states in Flink programs. By
+ * implementing the methods declared in this abstraction the state of the
+ * operator can be checkpointed by the fault tolerance mechanism.
+ *
+ * @param <T>
+ * The type of the operator state.
+ */
+public abstract class OperatorState<T> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ protected T state;
+
+ /**
+ * Constructor used for initializing the state. In case of failure, the
+ * state will be reinitialized using this constructor, then
+ * {@link #restore(StateCheckpoint)} will be used to restore from the last
+ * available backup.
+ */
+ public OperatorState() {
+ state = null;
+ }
+
+ /**
+ * Initializes the state using the given state object.
+ *
+ * @param initialState
+ * The initial state object
+ */
+ public OperatorState(T initialState) {
+ state = initialState;
+ }
+
+ /**
+ * Returns the currently stored state object.
+ *
+ * @return The state.
+ */
+ public T getState() {
+ return state;
+ }
+
+ /**
+ * Sets the current state object.
+ *
+ * @param state
+ * The new state object.
+ * @return The operator state with the new state object set.
+ */
+ public OperatorState<T> setState(T state) {
+ this.state = state;
+ return this;
+ }
+
+ /**
+ * Creates a {@link StateCheckpoint} that will be used to backup the state
+ * for failure recovery.
+ *
+ * @return The {@link StateCheckpoint} created.
+ */
+ public abstract StateCheckpoint<T> checkpoint();
+
+ /**
+ * Restores the state from the given {@link StateCheckpoint}.
+ *
+ * @param checkpoint
+ * The checkpoint to restore from
+ * @return The restored operator.
+ */
+ public abstract OperatorState<T> restore(StateCheckpoint<T> checkpoint);
+
+ @Override
+ public String toString() {
+ return state.toString();
+ }
+
+ public boolean stateEquals(OperatorState<T> other) {
+ return state.equals(other.state);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/PartitionableState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/PartitionableState.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/PartitionableState.java
new file mode 100644
index 0000000..c58c545
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/PartitionableState.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state;
+
+/**
+ * Base class for representing operator states that can be repartitioned for
+ * state state and load balancing.
+ *
+ * @param <T>
+ * The type of the operator state.
+ */
+public abstract class PartitionableState<T> extends OperatorState<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Repartitions(divides) the current state into the given number of new
+ * partitions. The created partitions will be used to redistribute then
+ * rebuild the state among the parallel instances of the operator. The
+ * implementation should reflect the partitioning of the input values to
+ * maintain correct operator behavior.
+ *
+ * </br> </br> It is also assumed that if we would {@link #reBuild} the
+ * repartitioned state we would basically get the same as before.
+ *
+ *
+ * @param numberOfPartitions
+ * The desired number of partitions. The method must return an
+ * array of that size.
+ * @return The array containing the state part for each partition.
+ */
+ public abstract OperatorState<T>[] repartition(int numberOfPartitions);
+
+ /**
+ * Rebuilds the current state partition from the given parts. Used for
+ * building the state after a re-balance phase.
+ *
+ * @param parts
+ * The state parts that will be used to rebuild the current
+ * partition.
+ * @return The rebuilt operator state.
+ */
+ public abstract OperatorState<T> reBuild(OperatorState<T>... parts);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/SimpleState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/SimpleState.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/SimpleState.java
new file mode 100644
index 0000000..7ae1f81
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/SimpleState.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state;
+
+import org.apache.flink.streaming.state.checkpoint.StateCheckpoint;
+
+/**
+ * Basic {@link OperatorState} for storing and updating simple objects. By default the
+ * whole stored object is checkpointed at each backup. Override checkpoint to
+ * allow a more fine-grained behavior.
+ *
+ * @param <T>
+ * The type of the stored state object.
+ */
+public class SimpleState<T> extends OperatorState<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ public SimpleState() {
+ super();
+ }
+
+ public SimpleState(T initialState) {
+ super(initialState);
+ }
+
+ @Override
+ public StateCheckpoint<T> checkpoint() {
+ return new StateCheckpoint<T>(this);
+ }
+
+ @Override
+ public OperatorState<T> restore(StateCheckpoint<T> checkpoint) {
+ this.state = checkpoint.getCheckpointedState();
+ return this;
+ }
+
+}
[45/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java
deleted file mode 100644
index c824ea1..0000000
--- a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.record.io.jdbc;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.junit.Assert;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.types.DoubleValue;
-import org.apache.flink.types.FloatValue;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.StringValue;
-import org.apache.flink.types.Value;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class JDBCOutputFormatTest {
- private JDBCInputFormat jdbcInputFormat;
- private JDBCOutputFormat jdbcOutputFormat;
-
- private static Connection conn;
-
- static final Value[][] dbData = {
- {new IntValue(1001), new StringValue("Java for dummies"), new StringValue("Tan Ah Teck"), new DoubleValue(11.11), new IntValue(11)},
- {new IntValue(1002), new StringValue("More Java for dummies"), new StringValue("Tan Ah Teck"), new DoubleValue(22.22), new IntValue(22)},
- {new IntValue(1003), new StringValue("More Java for more dummies"), new StringValue("Mohammad Ali"), new DoubleValue(33.33), new IntValue(33)},
- {new IntValue(1004), new StringValue("A Cup of Java"), new StringValue("Kumar"), new DoubleValue(44.44), new IntValue(44)},
- {new IntValue(1005), new StringValue("A Teaspoon of Java"), new StringValue("Kevin Jones"), new DoubleValue(55.55), new IntValue(55)}};
-
- @BeforeClass
- public static void setUpClass() {
- try {
- System.setProperty("derby.stream.error.field", "org.apache.flink.api.java.record.io.jdbc.DevNullLogStream.DEV_NULL");
- prepareDerbyInputDatabase();
- prepareDerbyOutputDatabase();
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- private static void cleanUpDerbyDatabases() {
- try {
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
- conn = DriverManager.getConnection(dbURL);
- Statement stat = conn.createStatement();
- stat.executeUpdate("DROP TABLE books");
- stat.executeUpdate("DROP TABLE newbooks");
- stat.close();
- conn.close();
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- private static void prepareDerbyInputDatabase() throws ClassNotFoundException {
- try {
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
- conn = DriverManager.getConnection(dbURL);
- createTableBooks();
- insertDataToSQLTables();
- conn.close();
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- Assert.fail();
- } catch (SQLException e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- private static void prepareDerbyOutputDatabase() throws ClassNotFoundException {
- try {
- String dbURL = "jdbc:derby:memory:ebookshop;create=true";
- Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
- conn = DriverManager.getConnection(dbURL);
- createTableNewBooks();
- conn.close();
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- Assert.fail();
- } catch (SQLException e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-
- private static void createTableBooks() throws SQLException {
- StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE books (");
- sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
- sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
- sqlQueryBuilder.append("qty INT DEFAULT NULL,");
- sqlQueryBuilder.append("PRIMARY KEY (id))");
-
- Statement stat = conn.createStatement();
- stat.executeUpdate(sqlQueryBuilder.toString());
- stat.close();
- }
-
- private static void createTableNewBooks() throws SQLException {
- StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE newbooks (");
- sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
- sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
- sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
- sqlQueryBuilder.append("qty INT DEFAULT NULL,");
- sqlQueryBuilder.append("PRIMARY KEY (id))");
-
- Statement stat = conn.createStatement();
- stat.executeUpdate(sqlQueryBuilder.toString());
- stat.close();
- }
-
- private static void insertDataToSQLTables() throws SQLException {
- StringBuilder sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
- sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
- sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
- sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
- sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
- sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
-
- Statement stat = conn.createStatement();
- stat.execute(sqlQueryBuilder.toString());
- stat.close();
- }
-
-
- @After
- public void tearDown() {
- jdbcOutputFormat = null;
- cleanUpDerbyDatabases();
- }
-
- @Test
- public void testJDBCOutputFormat() throws IOException {
- String sourceTable = "books";
- String targetTable = "newbooks";
- String driverPath = "org.apache.derby.jdbc.EmbeddedDriver";
- String dbUrl = "jdbc:derby:memory:ebookshop";
-
- Configuration cfg = new Configuration();
- cfg.setString("driver", driverPath);
- cfg.setString("url", dbUrl);
- cfg.setString("query", "insert into " + targetTable + " (id, title, author, price, qty) values (?,?,?,?,?)");
- cfg.setInteger("fields", 5);
- cfg.setClass("type0", IntValue.class);
- cfg.setClass("type1", StringValue.class);
- cfg.setClass("type2", StringValue.class);
- cfg.setClass("type3", FloatValue.class);
- cfg.setClass("type4", IntValue.class);
-
- jdbcOutputFormat = new JDBCOutputFormat();
- jdbcOutputFormat.configure(cfg);
- jdbcOutputFormat.open(0,1);
-
- jdbcInputFormat = new JDBCInputFormat(
- driverPath,
- dbUrl,
- "select * from " + sourceTable);
- jdbcInputFormat.configure(null);
-
- Record record = new Record();
- while (!jdbcInputFormat.reachedEnd()) {
- jdbcInputFormat.nextRecord(record);
- jdbcOutputFormat.writeRecord(record);
- }
-
- jdbcOutputFormat.close();
- jdbcInputFormat.close();
-
- jdbcInputFormat = new JDBCInputFormat(
- driverPath,
- dbUrl,
- "select * from " + targetTable);
- jdbcInputFormat.configure(null);
-
- int recordCount = 0;
- while (!jdbcInputFormat.reachedEnd()) {
- jdbcInputFormat.nextRecord(record);
- Assert.assertEquals(5, record.getNumFields());
- Assert.assertEquals("Field 0 should be int", IntValue.class, record.getField(0, IntValue.class).getClass());
- Assert.assertEquals("Field 1 should be String", StringValue.class, record.getField(1, StringValue.class).getClass());
- Assert.assertEquals("Field 2 should be String", StringValue.class, record.getField(2, StringValue.class).getClass());
- Assert.assertEquals("Field 3 should be float", DoubleValue.class, record.getField(3, DoubleValue.class).getClass());
- Assert.assertEquals("Field 4 should be int", IntValue.class, record.getField(4, IntValue.class).getClass());
-
- int[] pos = {0, 1, 2, 3, 4};
- Value[] values = {new IntValue(), new StringValue(), new StringValue(), new DoubleValue(), new IntValue()};
- Assert.assertTrue(record.equalsFields(pos, dbData[recordCount], values));
-
- recordCount++;
- }
- Assert.assertEquals(5, recordCount);
-
- jdbcInputFormat.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/resources/log4j-test.properties b/flink-addons/flink-jdbc/src/test/resources/log4j-test.properties
deleted file mode 100644
index 2fb9345..0000000
--- a/flink-addons/flink-jdbc/src/test/resources/log4j-test.properties
+++ /dev/null
@@ -1,19 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/resources/logback-test.xml b/flink-addons/flink-jdbc/src/test/resources/logback-test.xml
deleted file mode 100644
index 8b3bb27..0000000
--- a/flink-addons/flink-jdbc/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
- </encoder>
- </appender>
-
- <root level="WARN">
- <appender-ref ref="STDOUT"/>
- </root>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/pom.xml b/flink-addons/flink-spargel/pom.xml
deleted file mode 100644
index 863e66e..0000000
--- a/flink-addons/flink-spargel/pom.xml
+++ /dev/null
@@ -1,60 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-addons</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-spargel</artifactId>
- <name>flink-spargel</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessageIterator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessageIterator.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessageIterator.java
deleted file mode 100644
index 535732d..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessageIterator.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java;
-
-import java.util.Iterator;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-
-/**
- * An iterator that returns messages. The iterator is {@link java.lang.Iterable} at the same time to support
- * the <i>foreach</i> syntax.
- */
-public final class MessageIterator<Message> implements Iterator<Message>, Iterable<Message>, java.io.Serializable {
- private static final long serialVersionUID = 1L;
-
- private transient Iterator<Tuple2<?, Message>> source;
-
-
- final void setSource(Iterator<Tuple2<?, Message>> source) {
- this.source = source;
- }
-
- @Override
- public final boolean hasNext() {
- return this.source.hasNext();
- }
-
- @Override
- public final Message next() {
- return this.source.next().f1;
- }
-
- @Override
- public final void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Iterator<Message> iterator() {
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessagingFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessagingFunction.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessagingFunction.java
deleted file mode 100644
index 41c2720..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessagingFunction.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java;
-
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.Iterator;
-
-import org.apache.flink.api.common.aggregators.Aggregator;
-import org.apache.flink.api.common.functions.IterationRuntimeContext;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.spargel.java.OutgoingEdge;
-import org.apache.flink.types.Value;
-import org.apache.flink.util.Collector;
-
-/**
- * The base class for functions that produce messages between vertices as a part of a {@link VertexCentricIteration}.
- *
- * @param <VertexKey> The type of the vertex key (the vertex identifier).
- * @param <VertexValue> The type of the vertex value (the state of the vertex).
- * @param <Message> The type of the message sent between vertices along the edges.
- * @param <EdgeValue> The type of the values that are associated with the edges.
- */
-public abstract class MessagingFunction<VertexKey extends Comparable<VertexKey>, VertexValue, Message, EdgeValue> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- // --------------------------------------------------------------------------------------------
- // Public API Methods
- // --------------------------------------------------------------------------------------------
-
- /**
- * This method is invoked once per superstep for each vertex that was changed in that superstep.
- * It needs to produce the messages that will be received by vertices in the next superstep.
- *
- * @param vertexKey The key of the vertex that was changed.
- * @param vertexValue The value (state) of the vertex that was changed.
- *
- * @throws Exception The computation may throw exceptions, which causes the superstep to fail.
- */
- public abstract void sendMessages(VertexKey vertexKey, VertexValue vertexValue) throws Exception;
-
- /**
- * This method is executed one per superstep before the vertex update function is invoked for each vertex.
- *
- * @throws Exception Exceptions in the pre-superstep phase cause the superstep to fail.
- */
- public void preSuperstep() throws Exception {}
-
- /**
- * This method is executed one per superstep after the vertex update function has been invoked for each vertex.
- *
- * @throws Exception Exceptions in the post-superstep phase cause the superstep to fail.
- */
- public void postSuperstep() throws Exception {}
-
-
- /**
- * Gets an {@link java.lang.Iterable} with all outgoing edges. This method is mutually exclusive with
- * {@link #sendMessageToAllNeighbors(Object)} and may be called only once.
- *
- * @return An iterator with all outgoing edges.
- */
- @SuppressWarnings("unchecked")
- public Iterable<OutgoingEdge<VertexKey, EdgeValue>> getOutgoingEdges() {
- if (edgesUsed) {
- throw new IllegalStateException("Can use either 'getOutgoingEdges()' or 'sendMessageToAllTargets()' exactly once.");
- }
- edgesUsed = true;
-
- if (this.edgeWithValueIter != null) {
- this.edgeWithValueIter.set((Iterator<Tuple3<VertexKey, VertexKey, EdgeValue>>) edges);
- return this.edgeWithValueIter;
- } else {
- this.edgeNoValueIter.set((Iterator<Tuple2<VertexKey, VertexKey>>) edges);
- return this.edgeNoValueIter;
- }
- }
-
- /**
- * Sends the given message to all vertices that are targets of an outgoing edge of the changed vertex.
- * This method is mutually exclusive to the method {@link #getOutgoingEdges()} and may be called only once.
- *
- * @param m The message to send.
- */
- public void sendMessageToAllNeighbors(Message m) {
- if (edgesUsed) {
- throw new IllegalStateException("Can use either 'getOutgoingEdges()' or 'sendMessageToAllTargets()' exactly once.");
- }
-
- edgesUsed = true;
-
- outValue.f1 = m;
-
- while (edges.hasNext()) {
- Tuple next = (Tuple) edges.next();
- VertexKey k = next.getField(1);
- outValue.f0 = k;
- out.collect(outValue);
- }
- }
-
- /**
- * Sends the given message to the vertex identified by the given key. If the target vertex does not exist,
- * the next superstep will cause an exception due to a non-deliverable message.
- *
- * @param target The key (id) of the target vertex to message.
- * @param m The message.
- */
- public void sendMessageTo(VertexKey target, Message m) {
- outValue.f0 = target;
- outValue.f1 = m;
- out.collect(outValue);
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the number of the superstep, starting at <tt>1</tt>.
- *
- * @return The number of the current superstep.
- */
- public int getSuperstepNumber() {
- return this.runtimeContext.getSuperstepNumber();
- }
-
- /**
- * Gets the iteration aggregator registered under the given name. The iteration aggregator is combines
- * all aggregates globally once per superstep and makes them available in the next superstep.
- *
- * @param name The name of the aggregator.
- * @return The aggregator registered under this name, or null, if no aggregator was registered.
- */
- public <T extends Aggregator<?>> T getIterationAggregator(String name) {
- return this.runtimeContext.<T>getIterationAggregator(name);
- }
-
- /**
- * Get the aggregated value that an aggregator computed in the previous iteration.
- *
- * @param name The name of the aggregator.
- * @return The aggregated value of the previous iteration.
- */
- public <T extends Value> T getPreviousIterationAggregate(String name) {
- return this.runtimeContext.<T>getPreviousIterationAggregate(name);
- }
-
- /**
- * Gets the broadcast data set registered under the given name. Broadcast data sets
- * are available on all parallel instances of a function. They can be registered via
- * {@link VertexCentricIteration#addBroadcastSetForMessagingFunction(String, org.apache.flink.api.java.DataSet)}.
- *
- * @param name The name under which the broadcast set is registered.
- * @return The broadcast data set.
- */
- public <T> Collection<T> getBroadcastSet(String name) {
- return this.runtimeContext.<T>getBroadcastVariable(name);
- }
-
- // --------------------------------------------------------------------------------------------
- // internal methods and state
- // --------------------------------------------------------------------------------------------
-
- private Tuple2<VertexKey, Message> outValue;
-
- private IterationRuntimeContext runtimeContext;
-
- private Iterator<?> edges;
-
- private Collector<Tuple2<VertexKey, Message>> out;
-
- private EdgesIteratorNoEdgeValue<VertexKey, EdgeValue> edgeNoValueIter;
-
- private EdgesIteratorWithEdgeValue<VertexKey, EdgeValue> edgeWithValueIter;
-
- private boolean edgesUsed;
-
-
- void init(IterationRuntimeContext context, boolean hasEdgeValue) {
- this.runtimeContext = context;
- this.outValue = new Tuple2<VertexKey, Message>();
-
- if (hasEdgeValue) {
- this.edgeWithValueIter = new EdgesIteratorWithEdgeValue<VertexKey, EdgeValue>();
- } else {
- this.edgeNoValueIter = new EdgesIteratorNoEdgeValue<VertexKey, EdgeValue>();
- }
- }
-
- void set(Iterator<?> edges, Collector<Tuple2<VertexKey, Message>> out) {
- this.edges = edges;
- this.out = out;
- this.edgesUsed = false;
- }
-
-
-
- private static final class EdgesIteratorNoEdgeValue<VertexKey extends Comparable<VertexKey>, EdgeValue>
- implements Iterator<OutgoingEdge<VertexKey, EdgeValue>>, Iterable<OutgoingEdge<VertexKey, EdgeValue>>
- {
- private Iterator<Tuple2<VertexKey, VertexKey>> input;
-
- private OutgoingEdge<VertexKey, EdgeValue> edge = new OutgoingEdge<VertexKey, EdgeValue>();
-
-
- void set(Iterator<Tuple2<VertexKey, VertexKey>> input) {
- this.input = input;
- }
-
- @Override
- public boolean hasNext() {
- return input.hasNext();
- }
-
- @Override
- public OutgoingEdge<VertexKey, EdgeValue> next() {
- Tuple2<VertexKey, VertexKey> next = input.next();
- edge.set(next.f1, null);
- return edge;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Iterator<OutgoingEdge<VertexKey, EdgeValue>> iterator() {
- return this;
- }
- }
-
-
- private static final class EdgesIteratorWithEdgeValue<VertexKey extends Comparable<VertexKey>, EdgeValue>
- implements Iterator<OutgoingEdge<VertexKey, EdgeValue>>, Iterable<OutgoingEdge<VertexKey, EdgeValue>>
- {
- private Iterator<Tuple3<VertexKey, VertexKey, EdgeValue>> input;
-
- private OutgoingEdge<VertexKey, EdgeValue> edge = new OutgoingEdge<VertexKey, EdgeValue>();
-
- void set(Iterator<Tuple3<VertexKey, VertexKey, EdgeValue>> input) {
- this.input = input;
- }
-
- @Override
- public boolean hasNext() {
- return input.hasNext();
- }
-
- @Override
- public OutgoingEdge<VertexKey, EdgeValue> next() {
- Tuple3<VertexKey, VertexKey, EdgeValue> next = input.next();
- edge.set(next.f1, next.f2);
- return edge;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- @Override
- public Iterator<OutgoingEdge<VertexKey, EdgeValue>> iterator() {
- return this;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/OutgoingEdge.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/OutgoingEdge.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/OutgoingEdge.java
deleted file mode 100644
index 6d97525..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/OutgoingEdge.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java;
-
-/**
- * <tt>Edge</tt> objects represent edges between vertices. Edges are defined by their source and target
- * vertex id. Edges may have an associated value (for example a weight or a distance), if the
- * graph algorithm was initialized with the
- * {@link VertexCentricIteration#withValuedEdges(org.apache.flink.api.java.DataSet, VertexUpdateFunction, MessagingFunction, int)}
- * method.
- *
- * @param <VertexKey> The type of the vertex key.
- * @param <EdgeValue> The type of the value associated with the edge. For scenarios where the edges do not hold
- * value, this type may be arbitrary.
- */
-public final class OutgoingEdge<VertexKey extends Comparable<VertexKey>, EdgeValue> implements java.io.Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private VertexKey target;
-
- private EdgeValue edgeValue;
-
- void set(VertexKey target, EdgeValue edgeValue) {
- this.target = target;
- this.edgeValue = edgeValue;
- }
-
- /**
- * Gets the target vertex id.
- *
- * @return The target vertex id.
- */
- public VertexKey target() {
- return target;
- }
-
- /**
- * Gets the value associated with the edge. The value may be null if the iteration was initialized with
- * an edge data set without edge values.
- * Typical examples of edge values are weights or distances of the path represented by the edge.
- *
- * @return The value associated with the edge.
- */
- public EdgeValue edgeValue() {
- return edgeValue;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
deleted file mode 100644
index 4f84467..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
+++ /dev/null
@@ -1,599 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.Validate;
-import org.apache.flink.api.common.aggregators.Aggregator;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.common.functions.RichCoGroupFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.operators.CoGroupOperator;
-import org.apache.flink.api.java.operators.CustomUnaryOperation;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.util.Collector;
-
-/**
- * This class represents iterative graph computations, programmed in a vertex-centric perspective.
- * It is a special case of <i>Bulk Synchronous Parallel<i> computation. The paradigm has also been
- * implemented by Google's <i>Pregel</i> system and by <i>Apache Giraph</i>.
- * <p>
- * Vertex centric algorithms operate on graphs, which are defined through vertices and edges. The
- * algorithms send messages along the edges and update the state of vertices based on
- * the old state and the incoming messages. All vertices have an initial state.
- * The computation terminates once no vertex updates it state any more.
- * Additionally, a maximum number of iterations (supersteps) may be specified.
- * <p>
- * The computation is here represented by two functions:
- * <ul>
- * <li>The {@link VertexUpdateFunction} receives incoming messages and may updates the state for
- * the vertex. If a state is updated, messages are sent from this vertex. Initially, all vertices are
- * considered updated.</li>
- * <li>The {@link MessagingFunction} takes the new vertex state and sends messages along the outgoing
- * edges of the vertex. The outgoing edges may optionally have an associated value, such as a weight.</li>
- * </ul>
- * <p>
- * Vertex-centric graph iterations are instantiated by the
- * {@link #withPlainEdges(DataSet, VertexUpdateFunction, MessagingFunction, int)} method, or the
- * {@link #withValuedEdges(DataSet, VertexUpdateFunction, MessagingFunction, int)} method, depending on whether
- * the graph's edges are carrying values.
- *
- * @param <VertexKey> The type of the vertex key (the vertex identifier).
- * @param <VertexValue> The type of the vertex value (the state of the vertex).
- * @param <Message> The type of the message sent between vertices along the edges.
- * @param <EdgeValue> The type of the values that are associated with the edges.
- */
-public class VertexCentricIteration<VertexKey extends Comparable<VertexKey>, VertexValue, Message, EdgeValue>
- implements CustomUnaryOperation<Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, VertexValue>>
-{
- private final VertexUpdateFunction<VertexKey, VertexValue, Message> updateFunction;
-
- private final MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> messagingFunction;
-
- private final DataSet<Tuple2<VertexKey, VertexKey>> edgesWithoutValue;
-
- private final DataSet<Tuple3<VertexKey, VertexKey, EdgeValue>> edgesWithValue;
-
- private final Map<String, Aggregator<?>> aggregators;
-
- private final int maximumNumberOfIterations;
-
- private final List<Tuple2<String, DataSet<?>>> bcVarsUpdate = new ArrayList<Tuple2<String,DataSet<?>>>(4);
-
- private final List<Tuple2<String, DataSet<?>>> bcVarsMessaging = new ArrayList<Tuple2<String,DataSet<?>>>(4);
-
- private final TypeInformation<Message> messageType;
-
- private DataSet<Tuple2<VertexKey, VertexValue>> initialVertices;
-
- private String name;
-
- private int parallelism = -1;
-
- private boolean unmanagedSolutionSet;
-
- // ----------------------------------------------------------------------------------
-
- private VertexCentricIteration(VertexUpdateFunction<VertexKey, VertexValue, Message> uf,
- MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf,
- DataSet<Tuple2<VertexKey, VertexKey>> edgesWithoutValue,
- int maximumNumberOfIterations)
- {
- Validate.notNull(uf);
- Validate.notNull(mf);
- Validate.notNull(edgesWithoutValue);
- Validate.isTrue(maximumNumberOfIterations > 0, "The maximum number of iterations must be at least one.");
-
- // check that the edges are actually a valid tuple set of vertex key types
- TypeInformation<Tuple2<VertexKey, VertexKey>> edgesType = edgesWithoutValue.getType();
- Validate.isTrue(edgesType.isTupleType() && edgesType.getArity() == 2, "The edges data set (for edges without edge values) must consist of 2-tuples.");
-
- TupleTypeInfo<?> tupleInfo = (TupleTypeInfo<?>) edgesType;
- Validate.isTrue(tupleInfo.getTypeAt(0).equals(tupleInfo.getTypeAt(1))
- && Comparable.class.isAssignableFrom(tupleInfo.getTypeAt(0).getTypeClass()),
- "Both tuple fields (source and target vertex id) must be of the data type that represents the vertex key and implement the java.lang.Comparable interface.");
-
- this.updateFunction = uf;
- this.messagingFunction = mf;
- this.edgesWithoutValue = edgesWithoutValue;
- this.edgesWithValue = null;
- this.maximumNumberOfIterations = maximumNumberOfIterations;
- this.aggregators = new HashMap<String, Aggregator<?>>();
-
- this.messageType = getMessageType(mf);
- }
-
- private VertexCentricIteration(VertexUpdateFunction<VertexKey, VertexValue, Message> uf,
- MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf,
- DataSet<Tuple3<VertexKey, VertexKey, EdgeValue>> edgesWithValue,
- int maximumNumberOfIterations,
- boolean edgeHasValueMarker)
- {
- Validate.notNull(uf);
- Validate.notNull(mf);
- Validate.notNull(edgesWithValue);
- Validate.isTrue(maximumNumberOfIterations > 0, "The maximum number of iterations must be at least one.");
-
- // check that the edges are actually a valid tuple set of vertex key types
- TypeInformation<Tuple3<VertexKey, VertexKey, EdgeValue>> edgesType = edgesWithValue.getType();
- Validate.isTrue(edgesType.isTupleType() && edgesType.getArity() == 3, "The edges data set (for edges with edge values) must consist of 3-tuples.");
-
- TupleTypeInfo<?> tupleInfo = (TupleTypeInfo<?>) edgesType;
- Validate.isTrue(tupleInfo.getTypeAt(0).equals(tupleInfo.getTypeAt(1))
- && Comparable.class.isAssignableFrom(tupleInfo.getTypeAt(0).getTypeClass()),
- "The first two tuple fields (source and target vertex id) must be of the data type that represents the vertex key and implement the java.lang.Comparable interface.");
-
- Validate.isTrue(maximumNumberOfIterations > 0, "The maximum number of iterations must be at least one.");
-
- this.updateFunction = uf;
- this.messagingFunction = mf;
- this.edgesWithoutValue = null;
- this.edgesWithValue = edgesWithValue;
- this.maximumNumberOfIterations = maximumNumberOfIterations;
- this.aggregators = new HashMap<String, Aggregator<?>>();
-
- this.messageType = getMessageType(mf);
- }
-
- private TypeInformation<Message> getMessageType(MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf) {
- return TypeExtractor.createTypeInfo(MessagingFunction.class, mf.getClass(), 2, null, null);
- }
-
- /**
- * Registers a new aggregator. Aggregators registered here are available during the execution of the vertex updates
- * via {@link VertexUpdateFunction#getIterationAggregator(String)} and
- * {@link VertexUpdateFunction#getPreviousIterationAggregate(String)}.
- *
- * @param name The name of the aggregator, used to retrieve it and its aggregates during execution.
- * @param aggregator The aggregator.
- */
- public void registerAggregator(String name, Aggregator<?> aggregator) {
- this.aggregators.put(name, aggregator);
- }
-
- /**
- * Adds a data set as a broadcast set to the messaging function.
- *
- * @param name The name under which the broadcast data is available in the messaging function.
- * @param data The data set to be broadcasted.
- */
- public void addBroadcastSetForMessagingFunction(String name, DataSet<?> data) {
- this.bcVarsMessaging.add(new Tuple2<String, DataSet<?>>(name, data));
- }
-
- /**
- * Adds a data set as a broadcast set to the vertex update function.
- *
- * @param name The name under which the broadcast data is available in the vertex update function.
- * @param data The data set to be broadcasted.
- */
- public void addBroadcastSetForUpdateFunction(String name, DataSet<?> data) {
- this.bcVarsUpdate.add(new Tuple2<String, DataSet<?>>(name, data));
- }
-
- /**
- * Sets the name for the vertex-centric iteration. The name is displayed in logs and messages.
- *
- * @param name The name for the iteration.
- */
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * Gets the name from this vertex-centric iteration.
- *
- * @return The name of the iteration.
- */
- public String getName() {
- return name;
- }
-
- /**
- * Sets the degree of parallelism for the iteration.
- *
- * @param parallelism The degree of parallelism.
- */
- public void setParallelism(int parallelism) {
- Validate.isTrue(parallelism > 0 || parallelism == -1, "The degree of parallelism must be positive, or -1 (use default).");
- this.parallelism = parallelism;
- }
-
- /**
- * Gets the iteration's degree of parallelism.
- *
- * @return The iterations parallelism, or -1, if not set.
- */
- public int getParallelism() {
- return parallelism;
- }
-
- /**
- * Defines whether the solution set is kept in managed memory (Flink's internal way of keeping object
- * in serialized form) or as a simple object map.
- * By default, the solution set runs in managed memory.
- *
- * @param unmanaged True, to keep the solution set in unmanaged memory, false otherwise.
- */
- public void setSolutionSetUnmanagedMemory(boolean unmanaged) {
- this.unmanagedSolutionSet = unmanaged;
- }
-
- /**
- * Gets whether the solution set is kept in managed memory (Flink's internal way of keeping object
- * in serialized form) or as a simple object map.
- * By default, the solution set runs in managed memory.
- *
- * @return True, if the solution set is in unmanaged memory, false otherwise.
- */
- public boolean isSolutionSetUnmanagedMemory() {
- return this.unmanagedSolutionSet;
- }
-
- // --------------------------------------------------------------------------------------------
- // Custom Operator behavior
- // --------------------------------------------------------------------------------------------
-
- /**
- * Sets the input data set for this operator. In the case of this operator this input data set represents
- * the set of vertices with their initial state.
- *
- * @param inputData The input data set, which in the case of this operator represents the set of
- * vertices with their initial state.
- *
- * @see org.apache.flink.api.java.operators.CustomUnaryOperation#setInput(org.apache.flink.api.java.DataSet)
- */
- @Override
- public void setInput(DataSet<Tuple2<VertexKey, VertexValue>> inputData) {
- // sanity check that we really have two tuples
- TypeInformation<Tuple2<VertexKey, VertexValue>> inputType = inputData.getType();
- Validate.isTrue(inputType.isTupleType() && inputType.getArity() == 2, "The input data set (the initial vertices) must consist of 2-tuples.");
-
- // check that the key type here is the same as for the edges
- TypeInformation<VertexKey> keyType = ((TupleTypeInfo<?>) inputType).getTypeAt(0);
- TypeInformation<?> edgeType = edgesWithoutValue != null ? edgesWithoutValue.getType() : edgesWithValue.getType();
- TypeInformation<VertexKey> edgeKeyType = ((TupleTypeInfo<?>) edgeType).getTypeAt(0);
-
- Validate.isTrue(keyType.equals(edgeKeyType), "The first tuple field (the vertex id) of the input data set (the initial vertices) " +
- "must be the same data type as the first fields of the edge data set (the source vertex id). " +
- "Here, the key type for the vertex ids is '%s' and the key type for the edges is '%s'.", keyType, edgeKeyType);
-
- this.initialVertices = inputData;
- }
-
- /**
- * Creates the operator that represents this vertex-centric graph computation.
- *
- * @return The operator that represents this vertex-centric graph computation.
- */
- @Override
- public DataSet<Tuple2<VertexKey, VertexValue>> createResult() {
- if (this.initialVertices == null) {
- throw new IllegalStateException("The input data set has not been set.");
- }
-
- // prepare some type information
- TypeInformation<Tuple2<VertexKey, VertexValue>> vertexTypes = initialVertices.getType();
- TypeInformation<VertexKey> keyType = ((TupleTypeInfo<?>) initialVertices.getType()).getTypeAt(0);
- TypeInformation<Tuple2<VertexKey, Message>> messageTypeInfo = new TupleTypeInfo<Tuple2<VertexKey,Message>>(keyType, messageType);
-
- // set up the iteration operator
- final String name = (this.name != null) ? this.name :
- "Vertex-centric iteration (" + updateFunction + " | " + messagingFunction + ")";
- final int[] zeroKeyPos = new int[] {0};
-
- final DeltaIteration<Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, VertexValue>> iteration =
- this.initialVertices.iterateDelta(this.initialVertices, this.maximumNumberOfIterations, zeroKeyPos);
- iteration.name(name);
- iteration.parallelism(parallelism);
- iteration.setSolutionSetUnManaged(unmanagedSolutionSet);
-
- // register all aggregators
- for (Map.Entry<String, Aggregator<?>> entry : this.aggregators.entrySet()) {
- iteration.registerAggregator(entry.getKey(), entry.getValue());
- }
-
- // build the messaging function (co group)
- CoGroupOperator<?, ?, Tuple2<VertexKey, Message>> messages;
- if (edgesWithoutValue != null) {
- MessagingUdfNoEdgeValues<VertexKey, VertexValue, Message> messenger = new MessagingUdfNoEdgeValues<VertexKey, VertexValue, Message>(messagingFunction, messageTypeInfo);
- messages = this.edgesWithoutValue.coGroup(iteration.getWorkset()).where(0).equalTo(0).with(messenger);
- }
- else {
- MessagingUdfWithEdgeValues<VertexKey, VertexValue, Message, EdgeValue> messenger = new MessagingUdfWithEdgeValues<VertexKey, VertexValue, Message, EdgeValue>(messagingFunction, messageTypeInfo);
- messages = this.edgesWithValue.coGroup(iteration.getWorkset()).where(0).equalTo(0).with(messenger);
- }
-
- // configure coGroup message function with name and broadcast variables
- messages = messages.name("Messaging");
- for (Tuple2<String, DataSet<?>> e : this.bcVarsMessaging) {
- messages = messages.withBroadcastSet(e.f1, e.f0);
- }
-
- VertexUpdateUdf<VertexKey, VertexValue, Message> updateUdf = new VertexUpdateUdf<VertexKey, VertexValue, Message>(updateFunction, vertexTypes);
-
- // build the update function (co group)
- CoGroupOperator<?, ?, Tuple2<VertexKey, VertexValue>> updates =
- messages.coGroup(iteration.getSolutionSet()).where(0).equalTo(0).with(updateUdf);
-
- // configure coGroup update function with name and broadcast variables
- updates = updates.name("Vertex State Updates");
- for (Tuple2<String, DataSet<?>> e : this.bcVarsUpdate) {
- updates = updates.withBroadcastSet(e.f1, e.f0);
- }
-
- // let the operator know that we preserve the key field
- updates.withForwardedFieldsFirst("0").withForwardedFieldsSecond("0");
-
- return iteration.closeWith(updates, updates);
-
- }
-
- // --------------------------------------------------------------------------------------------
- // Constructor builders to avoid signature conflicts with generic type erasure
- // --------------------------------------------------------------------------------------------
-
- /**
- * Creates a new vertex-centric iteration operator for graphs where the edges are not associated with a value.
- *
- * @param edgesWithoutValue The data set containing edges. Edges are represented as 2-tuples: (source-id, target-id)
- * @param vertexUpdateFunction The function that updates the state of the vertices from the incoming messages.
- * @param messagingFunction The function that turns changed vertex states into messages along the edges.
- *
- * @param <VertexKey> The type of the vertex key (the vertex identifier).
- * @param <VertexValue> The type of the vertex value (the state of the vertex).
- * @param <Message> The type of the message sent between vertices along the edges.
- *
- * @return An in stance of the vertex-centric graph computation operator.
- */
- public static final <VertexKey extends Comparable<VertexKey>, VertexValue, Message>
- VertexCentricIteration<VertexKey, VertexValue, Message, ?> withPlainEdges(
- DataSet<Tuple2<VertexKey, VertexKey>> edgesWithoutValue,
- VertexUpdateFunction<VertexKey, VertexValue, Message> vertexUpdateFunction,
- MessagingFunction<VertexKey, VertexValue, Message, ?> messagingFunction,
- int maximumNumberOfIterations)
- {
- @SuppressWarnings("unchecked")
- MessagingFunction<VertexKey, VertexValue, Message, Object> tmf =
- (MessagingFunction<VertexKey, VertexValue, Message, Object>) messagingFunction;
-
- return new VertexCentricIteration<VertexKey, VertexValue, Message, Object>(vertexUpdateFunction, tmf, edgesWithoutValue, maximumNumberOfIterations);
- }
-
- /**
- * Creates a new vertex-centric iteration operator for graphs where the edges are associated with a value (such as
- * a weight or distance).
- *
- * @param edgesWithValue The data set containing edges. Edges are represented as 2-tuples: (source-id, target-id)
- * @param uf The function that updates the state of the vertices from the incoming messages.
- * @param mf The function that turns changed vertex states into messages along the edges.
- *
- * @param <VertexKey> The type of the vertex key (the vertex identifier).
- * @param <VertexValue> The type of the vertex value (the state of the vertex).
- * @param <Message> The type of the message sent between vertices along the edges.
- * @param <EdgeValue> The type of the values that are associated with the edges.
- *
- * @return An in stance of the vertex-centric graph computation operator.
- */
- public static final <VertexKey extends Comparable<VertexKey>, VertexValue, Message, EdgeValue>
- VertexCentricIteration<VertexKey, VertexValue, Message, EdgeValue> withValuedEdges(
- DataSet<Tuple3<VertexKey, VertexKey, EdgeValue>> edgesWithValue,
- VertexUpdateFunction<VertexKey, VertexValue, Message> uf,
- MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf,
- int maximumNumberOfIterations)
- {
- return new VertexCentricIteration<VertexKey, VertexValue, Message, EdgeValue>(uf, mf, edgesWithValue, maximumNumberOfIterations, true);
- }
-
- // --------------------------------------------------------------------------------------------
- // Wrapping UDFs
- // --------------------------------------------------------------------------------------------
-
- private static final class VertexUpdateUdf<VertexKey extends Comparable<VertexKey>, VertexValue, Message>
- extends RichCoGroupFunction<Tuple2<VertexKey, Message>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, VertexValue>>
- implements ResultTypeQueryable<Tuple2<VertexKey, VertexValue>>
- {
- private static final long serialVersionUID = 1L;
-
- private final VertexUpdateFunction<VertexKey, VertexValue, Message> vertexUpdateFunction;
-
- private final MessageIterator<Message> messageIter = new MessageIterator<Message>();
-
- private transient TypeInformation<Tuple2<VertexKey, VertexValue>> resultType;
-
-
- private VertexUpdateUdf(VertexUpdateFunction<VertexKey, VertexValue, Message> vertexUpdateFunction,
- TypeInformation<Tuple2<VertexKey, VertexValue>> resultType)
- {
- this.vertexUpdateFunction = vertexUpdateFunction;
- this.resultType = resultType;
- }
-
- @Override
- public void coGroup(Iterable<Tuple2<VertexKey, Message>> messages, Iterable<Tuple2<VertexKey, VertexValue>> vertex,
- Collector<Tuple2<VertexKey, VertexValue>> out)
- throws Exception
- {
- final Iterator<Tuple2<VertexKey, VertexValue>> vertexIter = vertex.iterator();
-
- if (vertexIter.hasNext()) {
- Tuple2<VertexKey, VertexValue> vertexState = vertexIter.next();
-
- @SuppressWarnings("unchecked")
- Iterator<Tuple2<?, Message>> downcastIter = (Iterator<Tuple2<?, Message>>) (Iterator<?>) messages.iterator();
- messageIter.setSource(downcastIter);
-
- vertexUpdateFunction.setOutput(vertexState, out);
- vertexUpdateFunction.updateVertex(vertexState.f0, vertexState.f1, messageIter);
- }
- else {
- final Iterator<Tuple2<VertexKey, Message>> messageIter = messages.iterator();
- if (messageIter.hasNext()) {
- String message = "Target vertex does not exist!.";
- try {
- Tuple2<VertexKey, Message> next = messageIter.next();
- message = "Target vertex '" + next.f0 + "' does not exist!.";
- } catch (Throwable t) {}
- throw new Exception(message);
- } else {
- throw new Exception();
- }
- }
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- if (getIterationRuntimeContext().getSuperstepNumber() == 1) {
- this.vertexUpdateFunction.init(getIterationRuntimeContext());
- }
- this.vertexUpdateFunction.preSuperstep();
- }
-
- @Override
- public void close() throws Exception {
- this.vertexUpdateFunction.postSuperstep();
- }
-
- @Override
- public TypeInformation<Tuple2<VertexKey, VertexValue>> getProducedType() {
- return this.resultType;
- }
- }
-
- /*
- * UDF that encapsulates the message sending function for graphs where the edges have no associated values.
- */
- private static final class MessagingUdfNoEdgeValues<VertexKey extends Comparable<VertexKey>, VertexValue, Message>
- extends RichCoGroupFunction<Tuple2<VertexKey, VertexKey>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, Message>>
- implements ResultTypeQueryable<Tuple2<VertexKey, Message>>
- {
- private static final long serialVersionUID = 1L;
-
- private final MessagingFunction<VertexKey, VertexValue, Message, ?> messagingFunction;
-
- private transient TypeInformation<Tuple2<VertexKey, Message>> resultType;
-
-
- private MessagingUdfNoEdgeValues(MessagingFunction<VertexKey, VertexValue, Message, ?> messagingFunction,
- TypeInformation<Tuple2<VertexKey, Message>> resultType)
- {
- this.messagingFunction = messagingFunction;
- this.resultType = resultType;
- }
-
- @Override
- public void coGroup(Iterable<Tuple2<VertexKey, VertexKey>> edges,
- Iterable<Tuple2<VertexKey, VertexValue>> state, Collector<Tuple2<VertexKey, Message>> out)
- throws Exception
- {
- final Iterator<Tuple2<VertexKey, VertexValue>> stateIter = state.iterator();
-
- if (stateIter.hasNext()) {
- Tuple2<VertexKey, VertexValue> newVertexState = stateIter.next();
- messagingFunction.set((Iterator<?>) edges.iterator(), out);
- messagingFunction.sendMessages(newVertexState.f0, newVertexState.f1);
- }
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- if (getIterationRuntimeContext().getSuperstepNumber() == 1) {
- this.messagingFunction.init(getIterationRuntimeContext(), false);
- }
-
- this.messagingFunction.preSuperstep();
- }
-
- @Override
- public void close() throws Exception {
- this.messagingFunction.postSuperstep();
- }
-
- @Override
- public TypeInformation<Tuple2<VertexKey, Message>> getProducedType() {
- return this.resultType;
- }
- }
-
- /*
- * UDF that encapsulates the message sending function for graphs where the edges have an associated value.
- */
- private static final class MessagingUdfWithEdgeValues<VertexKey extends Comparable<VertexKey>, VertexValue, Message, EdgeValue>
- extends RichCoGroupFunction<Tuple3<VertexKey, VertexKey, EdgeValue>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, Message>>
- implements ResultTypeQueryable<Tuple2<VertexKey, Message>>
- {
- private static final long serialVersionUID = 1L;
-
- private final MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> messagingFunction;
-
- private transient TypeInformation<Tuple2<VertexKey, Message>> resultType;
-
-
- private MessagingUdfWithEdgeValues(MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> messagingFunction,
- TypeInformation<Tuple2<VertexKey, Message>> resultType)
- {
- this.messagingFunction = messagingFunction;
- this.resultType = resultType;
- }
-
- @Override
- public void coGroup(Iterable<Tuple3<VertexKey, VertexKey, EdgeValue>> edges,
- Iterable<Tuple2<VertexKey, VertexValue>> state, Collector<Tuple2<VertexKey, Message>> out)
- throws Exception
- {
- final Iterator<Tuple2<VertexKey, VertexValue>> stateIter = state.iterator();
-
- if (stateIter.hasNext()) {
- Tuple2<VertexKey, VertexValue> newVertexState = stateIter.next();
- messagingFunction.set((Iterator<?>) edges.iterator(), out);
- messagingFunction.sendMessages(newVertexState.f0, newVertexState.f1);
- }
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- if (getIterationRuntimeContext().getSuperstepNumber() == 1) {
- this.messagingFunction.init(getIterationRuntimeContext(), true);
- }
-
- this.messagingFunction.preSuperstep();
- }
-
- @Override
- public void close() throws Exception {
- this.messagingFunction.postSuperstep();
- }
-
- @Override
- public TypeInformation<Tuple2<VertexKey, Message>> getProducedType() {
- return this.resultType;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexUpdateFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexUpdateFunction.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexUpdateFunction.java
deleted file mode 100644
index a5548b7..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexUpdateFunction.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java;
-
-import java.io.Serializable;
-import java.util.Collection;
-
-import org.apache.flink.api.common.aggregators.Aggregator;
-import org.apache.flink.api.common.functions.IterationRuntimeContext;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.types.Value;
-import org.apache.flink.util.Collector;
-
-/**
- * This class must be extended by functions that compute the state of the vertex depending on the old state and the
- * incoming messages. The central method is {@link #updateVertex(Comparable, Object, MessageIterator)}, which is
- * invoked once per vertex per superstep.
- *
- * <VertexKey> The vertex key type.
- * <VertexValue> The vertex value type.
- * <Message> The message type.
- */
-public abstract class VertexUpdateFunction<VertexKey extends Comparable<VertexKey>, VertexValue, Message> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- // --------------------------------------------------------------------------------------------
- // Public API Methods
- // --------------------------------------------------------------------------------------------
-
- /**
- * This method is invoked once per vertex per superstep. It receives the current state of the vertex, as well as
- * the incoming messages. It may set a new vertex state via {@link #setNewVertexValue(Object)}. If the vertex
- * state is changed, it will trigger the sending of messages via the {@link MessagingFunction}.
- *
- * @param vertexKey The key (identifier) of the vertex.
- * @param vertexValue The value (state) of the vertex.
- * @param inMessages The incoming messages to this vertex.
- *
- * @throws Exception The computation may throw exceptions, which causes the superstep to fail.
- */
- public abstract void updateVertex(VertexKey vertexKey, VertexValue vertexValue, MessageIterator<Message> inMessages) throws Exception;
-
- /**
- * This method is executed one per superstep before the vertex update function is invoked for each vertex.
- *
- * @throws Exception Exceptions in the pre-superstep phase cause the superstep to fail.
- */
- public void preSuperstep() throws Exception {}
-
- /**
- * This method is executed one per superstep after the vertex update function has been invoked for each vertex.
- *
- * @throws Exception Exceptions in the post-superstep phase cause the superstep to fail.
- */
- public void postSuperstep() throws Exception {}
-
- /**
- * Sets the new value of this vertex. Setting a new value triggers the sending of outgoing messages from this vertex.
- *
- * @param newValue The new vertex value.
- */
- public void setNewVertexValue(VertexValue newValue) {
- outVal.f1 = newValue;
- out.collect(outVal);
- }
-
- /**
- * Gets the number of the superstep, starting at <tt>1</tt>.
- *
- * @return The number of the current superstep.
- */
- public int getSuperstepNumber() {
- return this.runtimeContext.getSuperstepNumber();
- }
-
- /**
- * Gets the iteration aggregator registered under the given name. The iteration aggregator is combines
- * all aggregates globally once per superstep and makes them available in the next superstep.
- *
- * @param name The name of the aggregator.
- * @return The aggregator registered under this name, or null, if no aggregator was registered.
- */
- public <T extends Aggregator<?>> T getIterationAggregator(String name) {
- return this.runtimeContext.<T>getIterationAggregator(name);
- }
-
- /**
- * Get the aggregated value that an aggregator computed in the previous iteration.
- *
- * @param name The name of the aggregator.
- * @return The aggregated value of the previous iteration.
- */
- public <T extends Value> T getPreviousIterationAggregate(String name) {
- return this.runtimeContext.<T>getPreviousIterationAggregate(name);
- }
-
- /**
- * Gets the broadcast data set registered under the given name. Broadcast data sets
- * are available on all parallel instances of a function. They can be registered via
- * {@link VertexCentricIteration#addBroadcastSetForUpdateFunction(String, org.apache.flink.api.java.DataSet)}.
- *
- * @param name The name under which the broadcast set is registered.
- * @return The broadcast data set.
- */
- public <T> Collection<T> getBroadcastSet(String name) {
- return this.runtimeContext.<T>getBroadcastVariable(name);
- }
-
- // --------------------------------------------------------------------------------------------
- // internal methods
- // --------------------------------------------------------------------------------------------
-
- private IterationRuntimeContext runtimeContext;
-
- private Collector<Tuple2<VertexKey, VertexValue>> out;
-
- private Tuple2<VertexKey, VertexValue> outVal;
-
-
- void init(IterationRuntimeContext context) {
- this.runtimeContext = context;
- }
-
- void setOutput(Tuple2<VertexKey, VertexValue> val, Collector<Tuple2<VertexKey, VertexValue>> out) {
- this.out = out;
- this.outVal = val;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
deleted file mode 100644
index 7574dab..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java.examples;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.spargel.java.MessageIterator;
-import org.apache.flink.spargel.java.MessagingFunction;
-import org.apache.flink.spargel.java.VertexCentricIteration;
-import org.apache.flink.spargel.java.VertexUpdateFunction;
-import org.apache.flink.types.NullValue;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-
-@SuppressWarnings({"serial", "unchecked"})
-public class SpargelConnectedComponents {
-
- public static void main(String[] args) throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> vertexIds = env.generateSequence(0, 10);
- DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(0L, 2L), new Tuple2<Long, Long>(2L, 4L), new Tuple2<Long, Long>(4L, 8L),
- new Tuple2<Long, Long>(1L, 5L), new Tuple2<Long, Long>(3L, 7L), new Tuple2<Long, Long>(3L, 9L));
-
- DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
-
- DataSet<Tuple2<Long, Long>> result = initialVertices.runOperation(VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100));
-
- result.print();
- env.execute("Spargel Connected Components");
- }
-
- public static final class CCUpdater extends VertexUpdateFunction<Long, Long, Long> {
- @Override
- public void updateVertex(Long vertexKey, Long vertexValue, MessageIterator<Long> inMessages) {
- long min = Long.MAX_VALUE;
- for (long msg : inMessages) {
- min = Math.min(min, msg);
- }
- if (min < vertexValue) {
- setNewVertexValue(min);
- }
- }
- }
-
- public static final class CCMessager extends MessagingFunction<Long, Long, Long, NullValue> {
- @Override
- public void sendMessages(Long vertexId, Long componentId) {
- sendMessageToAllNeighbors(componentId);
- }
- }
-
- /**
- * A map function that takes a Long value and creates a 2-tuple out of it:
- * <pre>(Long value) -> (value, value)</pre>
- */
- public static final class IdAssigner implements MapFunction<Long, Tuple2<Long, Long>> {
- @Override
- public Tuple2<Long, Long> map(Long value) {
- return new Tuple2<Long, Long>(value, value);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
deleted file mode 100644
index fccb195..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java.examples;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.spargel.java.MessageIterator;
-import org.apache.flink.spargel.java.MessagingFunction;
-import org.apache.flink.spargel.java.OutgoingEdge;
-import org.apache.flink.spargel.java.VertexCentricIteration;
-import org.apache.flink.spargel.java.VertexUpdateFunction;
-import org.apache.flink.util.Collector;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-
-/**
- * An implementation of the basic PageRank algorithm in the vertex-centric API (spargel).
- * In this implementation, the edges carry a weight (the transition probability).
- */
-@SuppressWarnings("serial")
-public class SpargelPageRank {
-
- private static final double BETA = 0.85;
-
-
- public static void main(String[] args) throws Exception {
- final int numVertices = 100;
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- // enumerate some sample edges and assign an initial uniform probability (rank)
- DataSet<Tuple2<Long, Double>> intialRanks = env.generateSequence(1, numVertices)
- .map(new MapFunction<Long, Tuple2<Long, Double>>() {
- public Tuple2<Long, Double> map(Long value) {
- return new Tuple2<Long, Double>(value, 1.0/numVertices);
- }
- });
-
- // generate some random edges. the transition probability on each edge is 1/num-out-edges of the source vertex
- DataSet<Tuple3<Long, Long, Double>> edgesWithProbability = env.generateSequence(1, numVertices)
- .flatMap(new FlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
- public void flatMap(Long value, Collector<Tuple3<Long, Long, Double>> out) {
- int numOutEdges = (int) (Math.random() * (numVertices / 2));
- for (int i = 0; i < numOutEdges; i++) {
- long target = (long) (Math.random() * numVertices) + 1;
- out.collect(new Tuple3<Long, Long, Double>(value, target, 1.0/numOutEdges));
- }
- }
- });
-
- DataSet<Tuple2<Long, Double>> result = intialRanks.runOperation(
- VertexCentricIteration.withValuedEdges(edgesWithProbability,
- new VertexRankUpdater(numVertices, BETA), new RankMessenger(), 20));
-
- result.print();
- env.execute("Spargel PageRank");
- }
-
- /**
- * Function that updates the rank of a vertex by summing up the partial ranks from all incoming messages
- * and then applying the dampening formula.
- */
- public static final class VertexRankUpdater extends VertexUpdateFunction<Long, Double, Double> {
-
- private final long numVertices;
- private final double beta;
-
- public VertexRankUpdater(long numVertices, double beta) {
- this.numVertices = numVertices;
- this.beta = beta;
- }
-
- @Override
- public void updateVertex(Long vertexKey, Double vertexValue, MessageIterator<Double> inMessages) {
- double rankSum = 0.0;
- for (double msg : inMessages) {
- rankSum += msg;
- }
-
- // apply the dampening factor / random jump
- double newRank = (beta * rankSum) + (1-BETA)/numVertices;
- setNewVertexValue(newRank);
- }
- }
-
- /**
- * Distributes the rank of a vertex among all target vertices according to the transition probability,
- * which is associated with an edge as the edge value.
- */
- public static final class RankMessenger extends MessagingFunction<Long, Double, Double, Double> {
-
- @Override
- public void sendMessages(Long vertexId, Double newRank) {
- for (OutgoingEdge<Long, Double> edge : getOutgoingEdges()) {
- sendMessageTo(edge.target(), newRank * edge.edgeValue());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
deleted file mode 100644
index ae4ee95..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java.examples;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.spargel.java.MessageIterator;
-import org.apache.flink.spargel.java.MessagingFunction;
-import org.apache.flink.spargel.java.OutgoingEdge;
-import org.apache.flink.spargel.java.VertexCentricIteration;
-import org.apache.flink.spargel.java.VertexUpdateFunction;
-import org.apache.flink.util.Collector;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-
-/**
- * An implementation of the basic PageRank algorithm in the vertex-centric API (spargel).
- * In this implementation, the edges carry a weight (the transition probability).
- */
-@SuppressWarnings("serial")
-public class SpargelPageRankCountingVertices {
-
- private static final double BETA = 0.85;
-
-
- public static void main(String[] args) throws Exception {
- final int NUM_VERTICES = 100;
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- // a list of vertices
- DataSet<Long> vertices = env.generateSequence(1, NUM_VERTICES);
-
- // generate some random edges. the transition probability on each edge is 1/num-out-edges of the source vertex
- DataSet<Tuple3<Long, Long, Double>> edgesWithProbability = env.generateSequence(1, NUM_VERTICES)
- .flatMap(new FlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
- public void flatMap(Long value, Collector<Tuple3<Long, Long, Double>> out) {
- int numOutEdges = (int) (Math.random() * (NUM_VERTICES / 2));
- for (int i = 0; i < numOutEdges; i++) {
- long target = (long) (Math.random() * NUM_VERTICES) + 1;
- out.collect(new Tuple3<Long, Long, Double>(value, target, 1.0/numOutEdges));
- }
- }
- });
-
- // ---------- start of the algorithm ---------------
-
- // count the number of vertices
- DataSet<Long> count = vertices
- .map(new MapFunction<Long, Long>() {
- public Long map(Long value) {
- return 1L;
- }
- })
- .reduce(new ReduceFunction<Long>() {
- public Long reduce(Long value1, Long value2) {
- return value1 + value2;
- }
- });
-
- // enumerate some sample edges and assign an initial uniform probability (rank)
- DataSet<Tuple2<Long, Double>> intialRanks = vertices
- .map(new RichMapFunction<Long, Tuple2<Long, Double>>() {
-
- private long numVertices;
-
- @Override
- public void open(Configuration parameters) {
- numVertices = getRuntimeContext().<Long>getBroadcastVariable("count").iterator().next();
- }
-
- public Tuple2<Long, Double> map(Long value) {
- return new Tuple2<Long, Double>(value, 1.0/numVertices);
- }
- }).withBroadcastSet(count, "count");
-
-
- VertexCentricIteration<Long, Double, Double, Double> iteration = VertexCentricIteration.withValuedEdges(edgesWithProbability,
- new VertexRankUpdater(BETA), new RankMessenger(), 20);
- iteration.addBroadcastSetForUpdateFunction("count", count);
-
-
- DataSet<Tuple2<Long, Double>> result = intialRanks.runOperation(iteration);
-
- result.print();
- env.execute("Spargel PageRank");
- }
-
- /**
- * Function that updates the rank of a vertex by summing up the partial ranks from all incoming messages
- * and then applying the dampening formula.
- */
- public static final class VertexRankUpdater extends VertexUpdateFunction<Long, Double, Double> {
-
- private final double beta;
- private long numVertices;
-
- public VertexRankUpdater(double beta) {
- this.beta = beta;
- }
-
- @Override
- public void preSuperstep() {
- numVertices = this.<Long>getBroadcastSet("count").iterator().next();
- }
-
- @Override
- public void updateVertex(Long vertexKey, Double vertexValue, MessageIterator<Double> inMessages) {
- double rankSum = 0.0;
- for (double msg : inMessages) {
- rankSum += msg;
- }
-
- // apply the dampening factor / random jump
- double newRank = (beta * rankSum) + (1-BETA)/numVertices;
- setNewVertexValue(newRank);
- }
- }
-
- /**
- * Distributes the rank of a vertex among all target vertices according to the transition probability,
- * which is associated with an edge as the edge value.
- */
- public static final class RankMessenger extends MessagingFunction<Long, Double, Double, Double> {
-
- @Override
- public void sendMessages(Long vertexId, Double newRank) {
- for (OutgoingEdge<Long, Double> edge : getOutgoingEdges()) {
- sendMessageTo(edge.target(), newRank * edge.edgeValue());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/Edge.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/Edge.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/Edge.java
deleted file mode 100644
index 0c39688..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/Edge.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java.record;
-
-
-import org.apache.flink.types.Key;
-import org.apache.flink.types.Value;
-
-
-public final class Edge<VertexKey extends Key<VertexKey>, EdgeValue extends Value> {
-
- private VertexKey target;
- private EdgeValue edgeValue;
-
- void set(VertexKey target, EdgeValue edgeValue) {
- this.target = target;
- this.edgeValue = edgeValue;
- }
-
- public VertexKey target() {
- return target;
- }
-
- public EdgeValue edgeValue() {
- return edgeValue;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessageIterator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessageIterator.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessageIterator.java
deleted file mode 100644
index 03022d2..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessageIterator.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java.record;
-
-import java.util.Iterator;
-
-import org.apache.flink.types.Record;
-import org.apache.flink.types.Value;
-
-public final class MessageIterator<Message extends Value> implements Iterator<Message>, Iterable<Message> {
-
- private final Message instance;
- private Iterator<Record> source;
-
- public MessageIterator(Message instance) {
- this.instance = instance;
- }
-
- public final void setSource(Iterator<Record> source) {
- this.source = source;
- }
-
- @Override
- public final boolean hasNext() {
- return this.source.hasNext();
- }
-
- @Override
- public final Message next() {
- this.source.next().getFieldInto(1, this.instance);
- return this.instance;
- }
-
- @Override
- public final void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Iterator<Message> iterator() {
- return this;
- }
-}
[14/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
new file mode 100644
index 0000000..4fe356b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -0,0 +1,658 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.RichGroupReduceFunction;
+import org.apache.flink.api.common.functions.RichReduceFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
+import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
+import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
+import org.apache.flink.streaming.api.invokable.operator.WindowGroupReduceInvokable;
+import org.apache.flink.streaming.api.invokable.operator.WindowReduceInvokable;
+import org.apache.flink.streaming.api.windowing.helper.Time;
+import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+
+/**
+ * A {@link WindowedDataStream} represents a data stream that has been divided
+ * into windows (predefined chunks). User defined function such as
+ * {@link #reduce(ReduceFunction)}, {@link #reduceGroup(GroupReduceFunction)} or
+ * aggregations can be applied to the windows.
+ *
+ * @param <OUT>
+ * The output type of the {@link WindowedDataStream}
+ */
+public class WindowedDataStream<OUT> {
+
+ protected DataStream<OUT> dataStream;
+ protected boolean isGrouped;
+ protected boolean allCentral;
+ protected KeySelector<OUT, ?> keySelector;
+
+ protected List<WindowingHelper<OUT>> triggerHelpers;
+ protected List<WindowingHelper<OUT>> evictionHelpers;
+
+ protected LinkedList<TriggerPolicy<OUT>> userTriggers;
+ protected LinkedList<EvictionPolicy<OUT>> userEvicters;
+
+ protected WindowedDataStream(DataStream<OUT> dataStream, WindowingHelper<OUT>... policyHelpers) {
+ this.dataStream = dataStream.copy();
+ this.triggerHelpers = new ArrayList<WindowingHelper<OUT>>();
+ for (WindowingHelper<OUT> helper : policyHelpers) {
+ this.triggerHelpers.add(helper);
+ }
+
+ if (dataStream instanceof GroupedDataStream) {
+ this.isGrouped = true;
+ this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
+ // set all policies distributed
+ this.allCentral = false;
+
+ } else {
+ this.isGrouped = false;
+ // set all policies central
+ this.allCentral = true;
+ }
+ }
+
+ protected WindowedDataStream(DataStream<OUT> dataStream, List<TriggerPolicy<OUT>> triggers,
+ List<EvictionPolicy<OUT>> evicters) {
+ this.dataStream = dataStream.copy();
+
+ if (triggers != null) {
+ this.userTriggers = new LinkedList<TriggerPolicy<OUT>>();
+ this.userTriggers.addAll(triggers);
+ }
+
+ if (evicters != null) {
+ this.userEvicters = new LinkedList<EvictionPolicy<OUT>>();
+ this.userEvicters.addAll(evicters);
+ }
+
+ if (dataStream instanceof GroupedDataStream) {
+ this.isGrouped = true;
+ this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
+ // set all policies distributed
+ this.allCentral = false;
+
+ } else {
+ this.isGrouped = false;
+ // set all policies central
+ this.allCentral = true;
+ }
+ }
+
+ protected WindowedDataStream(WindowedDataStream<OUT> windowedDataStream) {
+ this.dataStream = windowedDataStream.dataStream.copy();
+ this.isGrouped = windowedDataStream.isGrouped;
+ this.keySelector = windowedDataStream.keySelector;
+ this.triggerHelpers = windowedDataStream.triggerHelpers;
+ this.evictionHelpers = windowedDataStream.evictionHelpers;
+ this.userTriggers = windowedDataStream.userTriggers;
+ this.userEvicters = windowedDataStream.userEvicters;
+ this.allCentral = windowedDataStream.allCentral;
+ }
+
+ public <F> F clean(F f) {
+ return dataStream.clean(f);
+ }
+
+ /**
+ * Defines the slide size (trigger frequency) for the windowed data stream.
+ * This controls how often the user defined function will be triggered on
+ * the window. </br></br> For example to get a window of 5 elements with a
+ * slide of 2 seconds use: </br></br>
+ * {@code ds.window(Count.of(5)).every(Time.of(2,TimeUnit.SECONDS))}
+ * </br></br> The user function in this case will be called on the 5 most
+ * recent elements every 2 seconds
+ *
+ * @param policyHelpers
+ * The policies that define the triggering frequency
+ *
+ * @return The windowed data stream with triggering set
+ */
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ public WindowedDataStream<OUT> every(WindowingHelper... policyHelpers) {
+ WindowedDataStream<OUT> ret = this.copy();
+ if (ret.evictionHelpers == null) {
+ ret.evictionHelpers = ret.triggerHelpers;
+ ret.triggerHelpers = new ArrayList<WindowingHelper<OUT>>();
+ }
+ for (WindowingHelper<OUT> helper : policyHelpers) {
+ ret.triggerHelpers.add(helper);
+ }
+ return ret;
+ }
+
+ /**
+ * Groups the elements of the {@link WindowedDataStream} by the given key
+ * positions. The window sizes (evictions) and slide sizes (triggers) will
+ * be calculated on the whole stream (in a central fashion), but the user
+ * defined functions will be applied on a per group basis. </br></br> To get
+ * windows and triggers on a per group basis apply the
+ * {@link DataStream#window} operator on an already grouped data stream.
+ *
+ * @param fields
+ * The position of the fields to group by.
+ * @return The grouped {@link WindowedDataStream}
+ */
+ public WindowedDataStream<OUT> groupBy(int... fields) {
+ WindowedDataStream<OUT> ret = this.copy();
+ ret.dataStream = ret.dataStream.groupBy(fields);
+ ret.isGrouped = true;
+ ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
+ return ret;
+ }
+
+ /**
+ * Groups the elements of the {@link WindowedDataStream} by the given field
+ * expressions. The window sizes (evictions) and slide sizes (triggers) will
+ * be calculated on the whole stream (in a central fashion), but the user
+ * defined functions will be applied on a per group basis. </br></br> To get
+ * windows and triggers on a per group basis apply the
+ * {@link DataStream#window} operator on an already grouped data stream.
+ * </br></br> A field expression is either the name of a public field or a
+ * getter method with parentheses of the stream's underlying type. A dot can
+ * be used to drill down into objects, as in
+ * {@code "field1.getInnerField2()" }.
+ *
+ * @param fields
+ * The fields to group by
+ * @return The grouped {@link WindowedDataStream}
+ */
+ public WindowedDataStream<OUT> groupBy(String... fields) {
+ WindowedDataStream<OUT> ret = this.copy();
+ ret.dataStream = ret.dataStream.groupBy(fields);
+ ret.isGrouped = true;
+ ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
+ return ret;
+ }
+
+ /**
+ * Groups the elements of the {@link WindowedDataStream} using the given
+ * {@link KeySelector}. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * {@link DataStream#window} operator on an already grouped data stream.
+ *
+ * @param keySelector
+ * The keySelector used to extract the key for grouping.
+ * @return The grouped {@link WindowedDataStream}
+ */
+ public WindowedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
+ WindowedDataStream<OUT> ret = this.copy();
+ ret.dataStream = ret.dataStream.groupBy(keySelector);
+ ret.isGrouped = true;
+ ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
+ return ret;
+ }
+
+ /**
+ * Applies a reduce transformation on the windowed data stream by reducing
+ * the current window at every trigger.The user can also extend the
+ * {@link RichReduceFunction} to gain access to other features provided by
+ * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ *
+ * @param reduceFunction
+ * The reduce function that will be applied to the windows.
+ * @return The transformed DataStream
+ */
+ public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reduceFunction) {
+ return dataStream.transform("Window-Reduce", getType(),
+ getReduceInvokable(reduceFunction));
+ }
+
+ /**
+ * Applies a reduceGroup transformation on the windowed data stream by
+ * reducing the current window at every trigger. In contrast with the
+ * standard binary reducer, with reduceGroup the user can access all
+ * elements of the window at the same time through the iterable interface.
+ * The user can also extend the {@link RichGroupReduceFunction} to gain
+ * access to other features provided by the
+ * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ *
+ * @param reduceFunction
+ * The reduce function that will be applied to the windows.
+ * @return The transformed DataStream
+ */
+ public <R> SingleOutputStreamOperator<R, ?> reduceGroup(
+ GroupReduceFunction<OUT, R> reduceFunction) {
+
+ TypeInformation<OUT> inType = getType();
+ TypeInformation<R> outType = TypeExtractor
+ .getGroupReduceReturnTypes(reduceFunction, inType);
+
+ return dataStream.transform("WindowReduce", outType,
+ getReduceGroupInvokable(reduceFunction));
+ }
+
+ /**
+ * Applies a reduceGroup transformation on the windowed data stream by
+ * reducing the current window at every trigger. In contrast with the
+ * standard binary reducer, with reduceGroup the user can access all
+ * elements of the window at the same time through the iterable interface.
+ * The user can also extend the {@link RichGroupReduceFunction} to gain
+ * access to other features provided by the
+ * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ * </br> </br> This version of reduceGroup uses user supplied
+ * typeinformation for serializaton. Use this only when the system is unable
+ * to detect type information using:
+ * {@link #reduceGroup(GroupReduceFunction)}
+ *
+ * @param reduceFunction
+ * The reduce function that will be applied to the windows.
+ * @return The transformed DataStream
+ */
+ public <R> SingleOutputStreamOperator<R, ?> reduceGroup(
+ GroupReduceFunction<OUT, R> reduceFunction, TypeInformation<R> outType) {
+
+ return dataStream.transform("Window-Reduce", outType,
+ getReduceGroupInvokable(reduceFunction));
+ }
+
+ /**
+ * Applies an aggregation that sums every window of the data stream at the
+ * given position.
+ *
+ * @param positionToSum
+ * The position in the tuple/array to sum
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> sum(int positionToSum) {
+ dataStream.checkFieldRange(positionToSum);
+ return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(positionToSum,
+ dataStream.getClassAtPos(positionToSum), getType()));
+ }
+
+ /**
+ * Applies an aggregation that sums every window of the pojo data stream at
+ * the given field for every window. </br></br> A field expression is either
+ * the name of a public field or a getter method with parentheses of the
+ * stream's underlying type. A dot can be used to drill down into objects,
+ * as in {@code "field1.getInnerField2()" }.
+ *
+ * @param positionToSum
+ * The field to sum
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> sum(String field) {
+ return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field, getType()));
+ }
+
+ /**
+ * Applies an aggregation that that gives the minimum value of every window
+ * of the data stream at the given position.
+ *
+ * @param positionToMin
+ * The position to minimize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> min(int positionToMin) {
+ dataStream.checkFieldRange(positionToMin);
+ return aggregate(ComparableAggregator.getAggregator(positionToMin, getType(),
+ AggregationType.MIN));
+ }
+
+ /**
+ * Applies an aggregation that that gives the minimum value of the pojo data
+ * stream at the given field expression for every window. </br></br>A field
+ * expression is either the name of a public field or a getter method with
+ * parentheses of the {@link DataStream}S underlying type. A dot can be used
+ * to drill down into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> min(String field) {
+ return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MIN,
+ false));
+ }
+
+ /**
+ * Applies an aggregation that gives the minimum element of every window of
+ * the data stream by the given position. If more elements have the same
+ * minimum value the operator returns the first element by default.
+ *
+ * @param positionToMinBy
+ * The position to minimize by
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
+ return this.minBy(positionToMinBy, true);
+ }
+
+ /**
+ * Applies an aggregation that gives the minimum element of every window of
+ * the data stream by the given position. If more elements have the same
+ * minimum value the operator returns the first element by default.
+ *
+ * @param positionToMinBy
+ * The position to minimize by
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(String positionToMinBy) {
+ return this.minBy(positionToMinBy, true);
+ }
+
+ /**
+ * Applies an aggregation that gives the minimum element of every window of
+ * the data stream by the given position. If more elements have the same
+ * minimum value the operator returns either the first or last one depending
+ * on the parameter setting.
+ *
+ * @param positionToMinBy
+ * The position to minimize
+ * @param first
+ * If true, then the operator return the first element with the
+ * minimum value, otherwise returns the last
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy, boolean first) {
+ dataStream.checkFieldRange(positionToMinBy);
+ return aggregate(ComparableAggregator.getAggregator(positionToMinBy, getType(),
+ AggregationType.MINBY, first));
+ }
+
+ /**
+ * Applies an aggregation that that gives the minimum element of the pojo
+ * data stream by the given field expression for every window. A field
+ * expression is either the name of a public field or a getter method with
+ * parentheses of the {@link DataStream}S underlying type. A dot can be used
+ * to drill down into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @param first
+ * If True then in case of field equality the first object will
+ * be returned
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(String field, boolean first) {
+ return aggregate(ComparableAggregator.getAggregator(field, getType(),
+ AggregationType.MINBY, first));
+ }
+
+ /**
+ * Applies an aggregation that gives the maximum value of every window of
+ * the data stream at the given position.
+ *
+ * @param positionToMax
+ * The position to maximize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> max(int positionToMax) {
+ dataStream.checkFieldRange(positionToMax);
+ return aggregate(ComparableAggregator.getAggregator(positionToMax, getType(),
+ AggregationType.MAX));
+ }
+
+ /**
+ * Applies an aggregation that that gives the maximum value of the pojo data
+ * stream at the given field expression for every window. A field expression
+ * is either the name of a public field or a getter method with parentheses
+ * of the {@link DataStream}S underlying type. A dot can be used to drill
+ * down into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> max(String field) {
+ return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MAX,
+ false));
+ }
+
+ /**
+ * Applies an aggregation that gives the maximum element of every window of
+ * the data stream by the given position. If more elements have the same
+ * maximum value the operator returns the first by default.
+ *
+ * @param positionToMaxBy
+ * The position to maximize by
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
+ return this.maxBy(positionToMaxBy, true);
+ }
+
+ /**
+ * Applies an aggregation that gives the maximum element of every window of
+ * the data stream by the given position. If more elements have the same
+ * maximum value the operator returns the first by default.
+ *
+ * @param positionToMaxBy
+ * The position to maximize by
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(String positionToMaxBy) {
+ return this.maxBy(positionToMaxBy, true);
+ }
+
+ /**
+ * Applies an aggregation that gives the maximum element of every window of
+ * the data stream by the given position. If more elements have the same
+ * maximum value the operator returns either the first or last one depending
+ * on the parameter setting.
+ *
+ * @param positionToMaxBy
+ * The position to maximize by
+ * @param first
+ * If true, then the operator return the first element with the
+ * maximum value, otherwise returns the last
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
+ dataStream.checkFieldRange(positionToMaxBy);
+ return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getType(),
+ AggregationType.MAXBY, first));
+ }
+
+ /**
+ * Applies an aggregation that that gives the maximum element of the pojo
+ * data stream by the given field expression for every window. A field
+ * expression is either the name of a public field or a getter method with
+ * parentheses of the {@link DataStream}S underlying type. A dot can be used
+ * to drill down into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @param first
+ * If True then in case of field equality the first object will
+ * be returned
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(String field, boolean first) {
+ return aggregate(ComparableAggregator.getAggregator(field, getType(),
+ AggregationType.MAXBY, first));
+ }
+
+ private SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregator) {
+ StreamInvokable<OUT, OUT> invokable = getReduceInvokable(aggregator);
+
+ SingleOutputStreamOperator<OUT, ?> returnStream = dataStream.transform("Window-Aggregation",
+ getType(), invokable);
+
+ return returnStream;
+ }
+
+ private LinkedList<TriggerPolicy<OUT>> getTriggers() {
+
+ LinkedList<TriggerPolicy<OUT>> triggers = new LinkedList<TriggerPolicy<OUT>>();
+
+ if (triggerHelpers != null) {
+ for (WindowingHelper<OUT> helper : triggerHelpers) {
+ triggers.add(helper.toTrigger());
+ }
+ }
+
+ if (userTriggers != null) {
+ triggers.addAll(userTriggers);
+ }
+
+ return triggers;
+
+ }
+
+ private LinkedList<EvictionPolicy<OUT>> getEvicters() {
+
+ LinkedList<EvictionPolicy<OUT>> evicters = new LinkedList<EvictionPolicy<OUT>>();
+
+ if (evictionHelpers != null) {
+ for (WindowingHelper<OUT> helper : evictionHelpers) {
+ evicters.add(helper.toEvict());
+ }
+ } else {
+ if (userEvicters == null) {
+ boolean notOnlyTime=false;
+ for (WindowingHelper<OUT> helper : triggerHelpers){
+ if (helper instanceof Time<?>){
+ evicters.add(helper.toEvict());
+ } else {
+ notOnlyTime=true;
+ }
+ }
+ if (notOnlyTime){
+ evicters.add(new TumblingEvictionPolicy<OUT>());
+ }
+ }
+ }
+
+ if (userEvicters != null) {
+ evicters.addAll(userEvicters);
+ }
+
+ return evicters;
+ }
+
+ private LinkedList<TriggerPolicy<OUT>> getCentralTriggers() {
+ LinkedList<TriggerPolicy<OUT>> cTriggers = new LinkedList<TriggerPolicy<OUT>>();
+ if (allCentral) {
+ cTriggers.addAll(getTriggers());
+ } else {
+ for (TriggerPolicy<OUT> trigger : getTriggers()) {
+ if (trigger instanceof TimeTriggerPolicy) {
+ cTriggers.add(trigger);
+ }
+ }
+ }
+ return cTriggers;
+ }
+
+ private LinkedList<CloneableTriggerPolicy<OUT>> getDistributedTriggers() {
+ LinkedList<CloneableTriggerPolicy<OUT>> dTriggers = null;
+
+ if (!allCentral) {
+ dTriggers = new LinkedList<CloneableTriggerPolicy<OUT>>();
+ for (TriggerPolicy<OUT> trigger : getTriggers()) {
+ if (!(trigger instanceof TimeTriggerPolicy)) {
+ dTriggers.add((CloneableTriggerPolicy<OUT>) trigger);
+ }
+ }
+ }
+
+ return dTriggers;
+ }
+
+ private LinkedList<CloneableEvictionPolicy<OUT>> getDistributedEvicters() {
+ LinkedList<CloneableEvictionPolicy<OUT>> evicters = null;
+
+ if (!allCentral) {
+ evicters = new LinkedList<CloneableEvictionPolicy<OUT>>();
+ for (EvictionPolicy<OUT> evicter : getEvicters()) {
+ evicters.add((CloneableEvictionPolicy<OUT>) evicter);
+ }
+ }
+
+ return evicters;
+ }
+
+ private LinkedList<EvictionPolicy<OUT>> getCentralEvicters() {
+ if (allCentral) {
+ return getEvicters();
+ } else {
+ return null;
+ }
+ }
+
+ private <R> StreamInvokable<OUT, R> getReduceGroupInvokable(GroupReduceFunction<OUT, R> reducer) {
+ StreamInvokable<OUT, R> invokable;
+ if (isGrouped) {
+ invokable = new GroupedWindowInvokable<OUT, R>(clean(reducer), keySelector,
+ getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers(),
+ getCentralEvicters());
+
+ } else {
+ invokable = new WindowGroupReduceInvokable<OUT, R>(clean(reducer), getTriggers(),
+ getEvicters());
+ }
+ return invokable;
+ }
+
+ private StreamInvokable<OUT, OUT> getReduceInvokable(ReduceFunction<OUT> reducer) {
+ StreamInvokable<OUT, OUT> invokable;
+ if (isGrouped) {
+ invokable = new GroupedWindowInvokable<OUT, OUT>(clean(reducer), keySelector,
+ getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers(),
+ getCentralEvicters());
+
+ } else {
+ invokable = new WindowReduceInvokable<OUT>(clean(reducer), getTriggers(), getEvicters());
+ }
+ return invokable;
+ }
+
+ /**
+ * Gets the output type.
+ *
+ * @return The output type.
+ */
+ public TypeInformation<OUT> getType() {
+ return dataStream.getType();
+ }
+
+ public DataStream<OUT> getDataStream() {
+ return dataStream;
+ }
+
+ protected WindowedDataStream<OUT> copy() {
+ return new WindowedDataStream<OUT>(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
new file mode 100644
index 0000000..03160c2
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream.temporaloperator;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.operators.CrossOperator;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.function.co.CrossWindowFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
+
+public class StreamCrossOperator<I1, I2> extends
+ TemporalOperator<I1, I2, StreamCrossOperator.CrossWindow<I1, I2>> {
+
+ public StreamCrossOperator(DataStream<I1> input1, DataStream<I2> input2) {
+ super(input1, input2);
+ }
+
+ @Override
+ protected CrossWindow<I1, I2> createNextWindowOperator() {
+
+ CrossWindowFunction<I1, I2, Tuple2<I1, I2>> crossWindowFunction = new CrossWindowFunction<I1, I2, Tuple2<I1, I2>>(
+ input1.clean(new CrossOperator.DefaultCrossFunction<I1, I2>()));
+
+ return new CrossWindow<I1, I2>(this, input1.connect(input2).addGeneralWindowCombine(
+ crossWindowFunction,
+ new TupleTypeInfo<Tuple2<I1, I2>>(input1.getType(), input2.getType()), windowSize,
+ slideInterval, timeStamp1, timeStamp2));
+ }
+
+ public static class CrossWindow<I1, I2> extends
+ SingleOutputStreamOperator<Tuple2<I1, I2>, CrossWindow<I1, I2>> implements
+ TemporalWindow<CrossWindow<I1, I2>> {
+
+ private StreamCrossOperator<I1, I2> op;
+
+ public CrossWindow(StreamCrossOperator<I1, I2> op, DataStream<Tuple2<I1, I2>> ds) {
+ super(ds);
+ this.op = op;
+ }
+
+ public CrossWindow<I1, I2> every(long length, TimeUnit timeUnit) {
+ return every(timeUnit.toMillis(length));
+ }
+
+ @SuppressWarnings("unchecked")
+ public CrossWindow<I1, I2> every(long length) {
+ ((CoWindowInvokable<I1, I2, ?>) streamGraph.getInvokable(id)).setSlideSize(length);
+ return this;
+ }
+
+ /**
+ * Finalizes a temporal Cross transformation by applying a
+ * {@link CrossFunction} to each pair of crossed elements.<br/>
+ * Each CrossFunction call returns exactly one element.
+ *
+ * @param function
+ * The CrossFunction that is called for each pair of crossed
+ * elements.
+ * @return The crossed data streams
+ *
+ */
+ public <R> SingleOutputStreamOperator<R, ?> with(CrossFunction<I1, I2, R> function) {
+ TypeInformation<R> outTypeInfo = TypeExtractor.getCrossReturnTypes(function,
+ op.input1.getType(), op.input2.getType());
+
+ CoWindowInvokable<I1, I2, R> invokable = new CoWindowInvokable<I1, I2, R>(
+ new CrossWindowFunction<I1, I2, R>(clean(function)), op.windowSize,
+ op.slideInterval, op.timeStamp1, op.timeStamp2);
+
+ streamGraph.setInvokable(id, invokable);
+
+ return setType(outTypeInfo);
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
new file mode 100644
index 0000000..d2b2032
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream.temporaloperator;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.Keys;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.function.co.JoinWindowFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
+import org.apache.flink.streaming.util.keys.KeySelectorUtil;
+
+public class StreamJoinOperator<I1, I2> extends
+ TemporalOperator<I1, I2, StreamJoinOperator.JoinWindow<I1, I2>> {
+
+ public StreamJoinOperator(DataStream<I1> input1, DataStream<I2> input2) {
+ super(input1, input2);
+ }
+
+ @Override
+ protected JoinWindow<I1, I2> createNextWindowOperator() {
+ return new JoinWindow<I1, I2>(this);
+ }
+
+ public static class JoinWindow<I1, I2> implements TemporalWindow<JoinWindow<I1, I2>> {
+
+ private StreamJoinOperator<I1, I2> op;
+ private TypeInformation<I1> type1;
+
+ private JoinWindow(StreamJoinOperator<I1, I2> operator) {
+ this.op = operator;
+ this.type1 = op.input1.getType();
+ }
+
+ /**
+ * Continues a temporal Join transformation. <br/>
+ * Defines the {@link Tuple} fields of the first join {@link DataStream}
+ * that should be used as join keys.<br/>
+ * <b>Note: Fields can only be selected as join keys on Tuple
+ * DataStreams.</b><br/>
+ *
+ * @param fields
+ * The indexes of the other Tuple fields of the first join
+ * DataStreams that should be used as keys.
+ * @return An incomplete Join transformation. Call
+ * {@link JoinPredicate#equalTo} to continue the Join.
+ */
+ public JoinPredicate<I1, I2> where(int... fields) {
+ return new JoinPredicate<I1, I2>(op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys<I1>(fields, type1), type1));
+ }
+
+ /**
+ * Continues a temporal join transformation. <br/>
+ * Defines the fields of the first join {@link DataStream} that should
+ * be used as grouping keys. Fields are the names of member fields of
+ * the underlying type of the data stream.
+ *
+ * @param fields
+ * The fields of the first join DataStream that should be
+ * used as keys.
+ * @return An incomplete Join transformation. Call
+ * {@link JoinPredicate#equalTo} to continue the Join.
+ */
+ public JoinPredicate<I1, I2> where(String... fields) {
+ return new JoinPredicate<I1, I2>(op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys<I1>(fields, type1), type1));
+ }
+
+ /**
+ * Continues a temporal Join transformation and defines a
+ * {@link KeySelector} function for the first join {@link DataStream}
+ * .</br> The KeySelector function is called for each element of the
+ * first DataStream and extracts a single key value on which the
+ * DataStream is joined. </br>
+ *
+ * @param keySelector
+ * The KeySelector function which extracts the key values
+ * from the DataStream on which it is joined.
+ * @return An incomplete Join transformation. Call
+ * {@link JoinPredicate#equalTo} to continue the Join.
+ */
+ public <K> JoinPredicate<I1, I2> where(KeySelector<I1, K> keySelector) {
+ return new JoinPredicate<I1, I2>(op, keySelector);
+ }
+
+ @Override
+ public JoinWindow<I1, I2> every(long length, TimeUnit timeUnit) {
+ return every(timeUnit.toMillis(length));
+ }
+
+ @Override
+ public JoinWindow<I1, I2> every(long length) {
+ op.slideInterval = length;
+ return this;
+ }
+
+ // ----------------------------------------------------------------------------------------
+
+ }
+
+ /**
+ * Intermediate step of a temporal Join transformation. <br/>
+ * To continue the Join transformation, select the join key of the second
+ * input {@link DataStream} by calling {@link JoinPredicate#equalTo}
+ *
+ */
+ public static class JoinPredicate<I1, I2> {
+
+ private StreamJoinOperator<I1, I2> op;
+ private KeySelector<I1, ?> keys1;
+ private KeySelector<I2, ?> keys2;
+ private TypeInformation<I2> type2;
+
+ private JoinPredicate(StreamJoinOperator<I1, I2> operator, KeySelector<I1, ?> keys1) {
+ this.op = operator;
+ this.keys1 = keys1;
+ this.type2 = op.input2.getType();
+ }
+
+ /**
+ * Creates a temporal Join transformation and defines the {@link Tuple}
+ * fields of the second join {@link DataStream} that should be used as
+ * join keys.<br/>
+ * </p> The resulting operator wraps each pair of joining elements in a
+ * Tuple2<I1,I2>(first, second). To use a different wrapping function
+ * use {@link JoinedStream#with(JoinFunction)}
+ *
+ * @param fields
+ * The indexes of the Tuple fields of the second join
+ * DataStream that should be used as keys.
+ * @return A streaming join operator. Call {@link JoinedStream#with} to
+ * apply a custom wrapping
+ */
+ public JoinedStream<I1, I2> equalTo(int... fields) {
+ keys2 = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<I2>(fields, type2),
+ type2);
+ return createJoinOperator();
+ }
+
+ /**
+ * Creates a temporal Join transformation and defines the fields of the
+ * second join {@link DataStream} that should be used as join keys. </p>
+ * The resulting operator wraps each pair of joining elements in a
+ * Tuple2<I1,I2>(first, second). To use a different wrapping function
+ * use {@link JoinedStream#with(JoinFunction)}
+ *
+ * @param fields
+ * The fields of the second join DataStream that should be
+ * used as keys.
+ * @return A streaming join operator. Call {@link JoinedStream#with} to
+ * apply a custom wrapping
+ */
+ public JoinedStream<I1, I2> equalTo(String... fields) {
+ this.keys2 = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<I2>(fields,
+ type2), type2);
+ return createJoinOperator();
+ }
+
+ /**
+ * Creates a temporal Join transformation and defines a
+ * {@link KeySelector} function for the second join {@link DataStream}
+ * .</br> The KeySelector function is called for each element of the
+ * second DataStream and extracts a single key value on which the
+ * DataStream is joined. </p> The resulting operator wraps each pair of
+ * joining elements in a Tuple2<I1,I2>(first, second). To use a
+ * different wrapping function use
+ * {@link JoinedStream#with(JoinFunction)}
+ *
+ *
+ * @param keySelector
+ * The KeySelector function which extracts the key values
+ * from the second DataStream on which it is joined.
+ * @return A streaming join operator. Call {@link JoinedStream#with} to
+ * apply a custom wrapping
+ */
+ public <K> JoinedStream<I1, I2> equalTo(KeySelector<I2, K> keySelector) {
+ this.keys2 = keySelector;
+ return createJoinOperator();
+ }
+
+ private JoinedStream<I1, I2> createJoinOperator() {
+
+ JoinFunction<I1, I2, Tuple2<I1, I2>> joinFunction = new DefaultJoinFunction<I1, I2>();
+
+ JoinWindowFunction<I1, I2, Tuple2<I1, I2>> joinWindowFunction = getJoinWindowFunction(
+ joinFunction, this);
+
+ TypeInformation<Tuple2<I1, I2>> outType = new TupleTypeInfo<Tuple2<I1, I2>>(
+ op.input1.getType(), op.input2.getType());
+
+ return new JoinedStream<I1, I2>(this, op.input1
+ .groupBy(keys1)
+ .connect(op.input2.groupBy(keys2))
+ .addGeneralWindowCombine(joinWindowFunction, outType, op.windowSize,
+ op.slideInterval, op.timeStamp1, op.timeStamp2));
+ }
+ }
+
+ public static class JoinedStream<I1, I2> extends
+ SingleOutputStreamOperator<Tuple2<I1, I2>, JoinedStream<I1, I2>> {
+ private final JoinPredicate<I1, I2> predicate;
+
+ private JoinedStream(JoinPredicate<I1, I2> predicate, DataStream<Tuple2<I1, I2>> ds) {
+ super(ds);
+ this.predicate = predicate;
+ }
+
+ /**
+ * Completes a stream join. </p> The resulting operator wraps each pair
+ * of joining elements using the user defined {@link JoinFunction}
+ *
+ * @return The joined data stream.
+ */
+ public <OUT> SingleOutputStreamOperator<OUT, ?> with(JoinFunction<I1, I2, OUT> joinFunction) {
+
+ TypeInformation<OUT> outType = TypeExtractor.getJoinReturnTypes(joinFunction,
+ predicate.op.input1.getType(), predicate.op.input2.getType());
+
+ CoWindowInvokable<I1, I2, OUT> invokable = new CoWindowInvokable<I1, I2, OUT>(
+ getJoinWindowFunction(joinFunction, predicate), predicate.op.windowSize,
+ predicate.op.slideInterval, predicate.op.timeStamp1, predicate.op.timeStamp2);
+
+ streamGraph.setInvokable(id, invokable);
+
+ return setType(outType);
+ }
+ }
+
+ public static final class DefaultJoinFunction<I1, I2> implements
+ JoinFunction<I1, I2, Tuple2<I1, I2>> {
+
+ private static final long serialVersionUID = 1L;
+ private final Tuple2<I1, I2> outTuple = new Tuple2<I1, I2>();
+
+ @Override
+ public Tuple2<I1, I2> join(I1 first, I2 second) throws Exception {
+ outTuple.f0 = first;
+ outTuple.f1 = second;
+ return outTuple;
+ }
+ }
+
+ public static <I1, I2, OUT> JoinWindowFunction<I1, I2, OUT> getJoinWindowFunction(
+ JoinFunction<I1, I2, OUT> joinFunction, JoinPredicate<I1, I2> predicate) {
+ return new JoinWindowFunction<I1, I2, OUT>(predicate.keys1, predicate.keys2, joinFunction);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java
new file mode 100644
index 0000000..f121dfa
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream.temporaloperator;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.windowing.helper.SystemTimestamp;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+
+public abstract class TemporalOperator<I1, I2, OP extends TemporalWindow<OP>> {
+
+ public final DataStream<I1> input1;
+ public final DataStream<I2> input2;
+
+ public long windowSize;
+ public long slideInterval;
+
+ public TimestampWrapper<I1> timeStamp1;
+ public TimestampWrapper<I2> timeStamp2;
+
+ public TemporalOperator(DataStream<I1> input1, DataStream<I2> input2) {
+ if (input1 == null || input2 == null) {
+ throw new NullPointerException();
+ }
+ this.input1 = input1.copy();
+ this.input2 = input2.copy();
+ }
+
+ /**
+ * Continues a temporal transformation.<br/>
+ * Defines the window size on which the two DataStreams will be transformed.
+ * To define sliding windows call {@link TemporalWindow#every} on the
+ * resulting operator.
+ *
+ * @param length
+ * The size of the window in milliseconds.
+ * @param timeUnit
+ * The unit if time to be used
+ * @return An incomplete temporal transformation.
+ */
+ @SuppressWarnings("unchecked")
+ public OP onWindow(long length, TimeUnit timeUnit) {
+ return onWindow(timeUnit.toMillis(length),
+ (TimestampWrapper<I1>) SystemTimestamp.getWrapper(),
+ (TimestampWrapper<I2>) SystemTimestamp.getWrapper());
+ }
+
+ /**
+ * Continues a temporal transformation.<br/>
+ * Defines the window size on which the two DataStreams will be
+ * transformed.To define sliding windows call {@link TemporalWindow#every}
+ * on the resulting operator.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @param timeStamp1
+ * The timestamp used to extract time from the elements of the
+ * first data stream.
+ * @param timeStamp2
+ * The timestamp used to extract time from the elements of the
+ * second data stream.
+ * @return An incomplete temporal transformation.
+ */
+ public OP onWindow(long length, Timestamp<I1> timeStamp1, Timestamp<I2> timeStamp2) {
+ return onWindow(length, timeStamp1, timeStamp2, 0);
+ }
+
+ /**
+ * Continues a temporal transformation.<br/>
+ * Defines the window size on which the two DataStreams will be
+ * transformed.To define sliding windows call {@link TemporalWindow#every}
+ * on the resulting operator.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @param timeStamp1
+ * The timestamp used to extract time from the elements of the
+ * first data stream.
+ * @param timeStamp2
+ * The timestamp used to extract time from the elements of the
+ * second data stream.
+ * @param startTime
+ * The start time to measure the first window
+ * @return An incomplete temporal transformation.
+ */
+ public OP onWindow(long length, Timestamp<I1> timeStamp1, Timestamp<I2> timeStamp2,
+ long startTime) {
+ return onWindow(length, new TimestampWrapper<I1>(timeStamp1, startTime),
+ new TimestampWrapper<I2>(timeStamp2, startTime));
+ }
+
+ private OP onWindow(long length, TimestampWrapper<I1> timeStamp1,
+ TimestampWrapper<I2> timeStamp2) {
+
+ this.windowSize = length;
+ this.slideInterval = length;
+
+ this.timeStamp1 = timeStamp1;
+ this.timeStamp2 = timeStamp2;
+
+ return createNextWindowOperator();
+ }
+
+ protected abstract OP createNextWindowOperator();
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java
new file mode 100644
index 0000000..8ac1492
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream.temporaloperator;
+
+import java.util.concurrent.TimeUnit;
+
+public interface TemporalWindow<T> {
+
+ /**
+ * Defines the slide interval for this temporal operator
+ *
+ * @param length
+ * Length of the window
+ * @param timeUnit
+ * Unit of time
+ * @return The temporal operator with slide interval specified
+ */
+ public T every(long length, TimeUnit timeUnit);
+
+ /**
+ * Defines the slide interval for this temporal operator
+ *
+ * @param length
+ * Length of the window
+ * @return The temporal operator with slide interval specified
+ */
+ public T every(long length);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
new file mode 100755
index 0000000..4824fca
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.environment;
+
+import org.apache.flink.streaming.util.ClusterUtil;
+
+public class LocalStreamEnvironment extends StreamExecutionEnvironment {
+
+ protected static ClassLoader userClassLoader;
+
+ /**
+ * Executes the JobGraph of the on a mini cluster of CLusterUtil with a
+ * default name.
+ */
+ @Override
+ public void execute() throws Exception {
+ ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(), getDegreeOfParallelism());
+ }
+
+ /**
+ * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
+ * specified name.
+ *
+ * @param jobName
+ * name of the job
+ */
+ @Override
+ public void execute(String jobName) throws Exception {
+ ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(jobName),
+ getDegreeOfParallelism());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
new file mode 100644
index 0000000..2eb05ad
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.environment;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.client.program.Client;
+import org.apache.flink.client.program.JobWithJars;
+import org.apache.flink.client.program.ProgramInvocationException;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RemoteStreamEnvironment extends StreamExecutionEnvironment {
+ private static final Logger LOG = LoggerFactory.getLogger(RemoteStreamEnvironment.class);
+
+ private String host;
+ private int port;
+ private List<File> jarFiles;
+
+ /**
+ * Creates a new RemoteStreamEnvironment that points to the master
+ * (JobManager) described by the given host name and port.
+ *
+ * @param host
+ * The host name or address of the master (JobManager), where the
+ * program should be executed.
+ * @param port
+ * The port of the master (JobManager), where the program should
+ * be executed.
+ * @param jarFiles
+ * The JAR files with code that needs to be shipped to the
+ * cluster. If the program uses user-defined functions,
+ * user-defined input formats, or any libraries, those must be
+ * provided in the JAR files.
+ */
+ public RemoteStreamEnvironment(String host, int port, String... jarFiles) {
+ if (host == null) {
+ throw new NullPointerException("Host must not be null.");
+ }
+
+ if (port < 1 || port >= 0xffff) {
+ throw new IllegalArgumentException("Port out of range");
+ }
+
+ this.host = host;
+ this.port = port;
+ this.jarFiles = new ArrayList<File>();
+ for (int i = 0; i < jarFiles.length; i++) {
+ File file = new File(jarFiles[i]);
+ try {
+ JobWithJars.checkJarFile(file);
+ } catch (IOException e) {
+ throw new RuntimeException("Problem with jar file " + file.getAbsolutePath(), e);
+ }
+ this.jarFiles.add(file);
+ }
+ }
+
+ @Override
+ public void execute() {
+
+ JobGraph jobGraph = streamGraph.getJobGraph();
+ executeRemotely(jobGraph);
+ }
+
+ @Override
+ public void execute(String jobName) {
+
+ JobGraph jobGraph = streamGraph.getJobGraph(jobName);
+ executeRemotely(jobGraph);
+ }
+
+ /**
+ * Executes the remote job.
+ *
+ * @param jobGraph
+ * jobGraph to execute
+ */
+ private void executeRemotely(JobGraph jobGraph) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Running remotely at {}:{}", host, port);
+ }
+
+ for (File file : jarFiles) {
+ jobGraph.addJar(new Path(file.getAbsolutePath()));
+ }
+
+ Configuration configuration = jobGraph.getJobConfiguration();
+ Client client = new Client(new InetSocketAddress(host, port), configuration,
+ JobWithJars.buildUserCodeClassLoader(jarFiles, JobWithJars.class.getClassLoader()));
+
+ try {
+ client.run(jobGraph, true);
+ } catch (ProgramInvocationException e) {
+ throw new RuntimeException("Cannot execute job due to ProgramInvocationException", e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Remote Environment (" + this.host + ":" + this.port + " - DOP = "
+ + (getDegreeOfParallelism() == -1 ? "default" : getDegreeOfParallelism()) + ")";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
new file mode 100644
index 0000000..7d41d2a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.environment;
+
+import java.io.File;
+import java.util.List;
+
+import org.apache.flink.client.program.Client;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+
+public class StreamContextEnvironment extends StreamExecutionEnvironment {
+
+ protected static ClassLoader userClassLoader;
+ protected List<File> jars;
+ protected Client client;
+
+ protected StreamContextEnvironment(Client client, List<File> jars, int dop) {
+ this.client = client;
+ this.jars = jars;
+ if (dop > 0) {
+ setDegreeOfParallelism(dop);
+ } else {
+ setDegreeOfParallelism(GlobalConfiguration.getInteger(
+ ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE_KEY,
+ ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE));
+ }
+ }
+
+ @Override
+ public void execute() throws Exception {
+ execute(null);
+ }
+
+ @Override
+ public void execute(String jobName) throws Exception {
+ currentEnvironment = null;
+
+ JobGraph jobGraph;
+ if (jobName == null) {
+ jobGraph = this.streamGraph.getJobGraph();
+ } else {
+ jobGraph = this.streamGraph.getJobGraph(jobName);
+ }
+
+ for (File file : jars) {
+ jobGraph.addJar(new Path(file.getAbsolutePath()));
+ }
+
+ try {
+ client.run(jobGraph, true);
+
+ } catch (Exception e) {
+ throw e;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
new file mode 100644
index 0000000..45c14c1
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -0,0 +1,618 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.environment;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.Validate;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.TextInputFormat;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.client.program.Client;
+import org.apache.flink.client.program.Client.OptimizerPlanEnvironment;
+import org.apache.flink.client.program.ContextEnvironment;
+import org.apache.flink.client.program.PackagedProgram.PreviewPlanEnvironment;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.streaming.api.StreamGraph;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.function.source.FileMonitoringFunction;
+import org.apache.flink.streaming.api.function.source.FileMonitoringFunction.WatchType;
+import org.apache.flink.streaming.api.function.source.FileReadFunction;
+import org.apache.flink.streaming.api.function.source.FileSourceFunction;
+import org.apache.flink.streaming.api.function.source.FromElementsFunction;
+import org.apache.flink.streaming.api.function.source.GenSequenceFunction;
+import org.apache.flink.streaming.api.function.source.GenericSourceFunction;
+import org.apache.flink.streaming.api.function.source.ParallelSourceFunction;
+import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.api.function.source.SocketTextStreamFunction;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.invokable.SourceInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+
+/**
+ * {@link ExecutionEnvironment} for streaming jobs. An instance of it is
+ * necessary to construct streaming topologies.
+ *
+ */
+public abstract class StreamExecutionEnvironment {
+
+ private static int defaultLocalDop = Runtime.getRuntime().availableProcessors();
+
+ private long bufferTimeout = 100;
+
+ private ExecutionConfig config = new ExecutionConfig();
+
+ protected static StreamExecutionEnvironment currentEnvironment;
+
+ protected StreamGraph streamGraph;
+
+ // --------------------------------------------------------------------------------------------
+ // Constructor and Properties
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Constructor for creating StreamExecutionEnvironment
+ */
+ protected StreamExecutionEnvironment() {
+ streamGraph = new StreamGraph();
+ }
+
+ /**
+ * Sets the config object.
+ */
+ public void setConfig(ExecutionConfig config) {
+ Validate.notNull(config);
+ this.config = config;
+ }
+
+ /**
+ * Gets the config object.
+ */
+ public ExecutionConfig getConfig() {
+ return config;
+ }
+
+ /**
+ * Gets the degree of parallelism with which operation are executed by
+ * default. Operations can individually override this value to use a
+ * specific degree of parallelism.
+ *
+ * @return The degree of parallelism used by operations, unless they
+ * override that value.
+ */
+ public int getDegreeOfParallelism() {
+ return config.getDegreeOfParallelism();
+ }
+
+ /**
+ * Sets the degree of parallelism (DOP) for operations executed through this
+ * environment. Setting a DOP of x here will cause all operators (such as
+ * map, batchReduce) to run with x parallel instances. This method overrides
+ * the default parallelism for this environment. The
+ * {@link LocalStreamEnvironment} uses by default a value equal to the
+ * number of hardware contexts (CPU cores / threads). When executing the
+ * program via the command line client from a JAR file, the default degree
+ * of parallelism is the one configured for that setup.
+ *
+ * @param degreeOfParallelism
+ * The degree of parallelism
+ */
+ public StreamExecutionEnvironment setDegreeOfParallelism(int degreeOfParallelism) {
+ if (degreeOfParallelism < 1) {
+ throw new IllegalArgumentException("Degree of parallelism must be at least one.");
+ }
+ config.setDegreeOfParallelism(degreeOfParallelism);
+ return this;
+ }
+
+ /**
+ * Sets the maximum time frequency (milliseconds) for the flushing of the
+ * output buffers. By default the output buffers flush frequently to provide
+ * low latency and to aid smooth developer experience. Setting the parameter
+ * can result in three logical modes:
+ *
+ * <ul>
+ * <li>
+ * A positive integer triggers flushing periodically by that integer</li>
+ * <li>
+ * 0 triggers flushing after every record thus minimizing latency</li>
+ * <li>
+ * -1 triggers flushing only when the output buffer is full thus maximizing
+ * throughput</li>
+ * </ul>
+ *
+ * @param timeoutMillis
+ * The maximum time between two output flushes.
+ */
+ public StreamExecutionEnvironment setBufferTimeout(long timeoutMillis) {
+ if (timeoutMillis < -1) {
+ throw new IllegalArgumentException("Timeout of buffer must be non-negative or -1");
+ }
+
+ this.bufferTimeout = timeoutMillis;
+ return this;
+ }
+
+ /**
+ * Sets the maximum time frequency (milliseconds) for the flushing of the
+ * output buffers. For clarification on the extremal values see
+ * {@link #setBufferTimeout(long)}.
+ *
+ * @return The timeout of the buffer.
+ */
+ public long getBufferTimeout() {
+ return this.bufferTimeout;
+ }
+
+ /**
+ * Sets the default parallelism that will be used for the local execution
+ * environment created by {@link #createLocalEnvironment()}.
+ *
+ * @param degreeOfParallelism
+ * The degree of parallelism to use as the default local
+ * parallelism.
+ */
+ public static void setDefaultLocalParallelism(int degreeOfParallelism) {
+ defaultLocalDop = degreeOfParallelism;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Data stream creations
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Creates a DataStream that represents the Strings produced by reading the
+ * given file line wise. The file will be read with the system's default
+ * character set.
+ *
+ * @param filePath
+ * The path of the file, as a URI (e.g.,
+ * "file:///some/local/file" or "hdfs://host:port/file/path").
+ * @return The DataStream representing the text file.
+ */
+ public DataStreamSource<String> readTextFile(String filePath) {
+ Validate.notNull(filePath, "The file path may not be null.");
+ TextInputFormat format = new TextInputFormat(new Path(filePath));
+ TypeInformation<String> typeInfo = BasicTypeInfo.STRING_TYPE_INFO;
+
+ return addFileSource(format, typeInfo);
+ }
+
+ /**
+ * Creates a DataStream that represents the Strings produced by reading the
+ * given file line wise. The file will be read with the given character set.
+ *
+ * @param filePath
+ * The path of the file, as a URI (e.g.,
+ * "file:///some/local/file" or "hdfs://host:port/file/path").
+ * @return The DataStream representing the text file.
+ */
+ public DataStreamSource<String> readTextFile(String filePath, String charsetName) {
+ Validate.notNull(filePath, "The file path may not be null.");
+ TextInputFormat format = new TextInputFormat(new Path(filePath));
+ TypeInformation<String> typeInfo = BasicTypeInfo.STRING_TYPE_INFO;
+ format.setCharsetName(charsetName);
+
+ return addFileSource(format, typeInfo);
+ }
+
+ /**
+ * Creates a DataStream that contains the contents of file created while
+ * system watches the given path. The file will be read with the system's
+ * default character set.
+ *
+ * @param filePath
+ * The path of the file, as a URI (e.g.,
+ * "file:///some/local/file" or "hdfs://host:port/file/path/").
+ * @param intervalMillis
+ * The interval of file watching in milliseconds.
+ * @param watchType
+ * The watch type of file stream. When watchType is
+ * {@link WatchType.ONLY_NEW_FILES}, the system processes only
+ * new files. {@link WatchType.REPROCESS_WITH_APPENDED} means
+ * that the system re-processes all contents of appended file.
+ * {@link WatchType.PROCESS_ONLY_APPENDED} means that the system
+ * processes only appended contents of files.
+ *
+ * @return The DataStream containing the given directory.
+ */
+ public DataStream<String> readFileStream(String filePath, long intervalMillis,
+ WatchType watchType) {
+ DataStream<Tuple3<String, Long, Long>> source = addSource(new FileMonitoringFunction(
+ filePath, intervalMillis, watchType), null, "File Stream");
+
+ return source.flatMap(new FileReadFunction());
+ }
+
+ /**
+ * Creates a new DataStream that contains the given elements. The elements
+ * must all be of the same type, for example, all of the String or Integer.
+ * The sequence of elements must not be empty. Furthermore, the elements
+ * must be serializable (as defined in java.io.Serializable), because the
+ * execution environment may ship the elements into the cluster.
+ *
+ * @param data
+ * The collection of elements to create the DataStream from.
+ * @param <OUT>
+ * type of the returned stream
+ * @return The DataStream representing the elements.
+ */
+ public <OUT extends Serializable> DataStreamSource<OUT> fromElements(OUT... data) {
+ if (data.length == 0) {
+ throw new IllegalArgumentException(
+ "fromElements needs at least one element as argument");
+ }
+
+ TypeInformation<OUT> outTypeInfo = TypeExtractor.getForObject(data[0]);
+
+ SourceFunction<OUT> function = new FromElementsFunction<OUT>(data);
+
+ return addSource(function, outTypeInfo, "Elements source");
+ }
+
+ /**
+ * Creates a DataStream from the given non-empty collection. The type of the
+ * DataStream is that of the elements in the collection. The elements need
+ * to be serializable (as defined by java.io.Serializable), because the
+ * framework may move the elements into the cluster if needed.
+ *
+ * @param data
+ * The collection of elements to create the DataStream from.
+ * @param <OUT>
+ * type of the returned stream
+ * @return The DataStream representing the elements.
+ */
+ public <OUT extends Serializable> DataStreamSource<OUT> fromCollection(Collection<OUT> data) {
+ if (data == null) {
+ throw new NullPointerException("Collection must not be null");
+ }
+
+ if (data.isEmpty()) {
+ throw new IllegalArgumentException("Collection must not be empty");
+ }
+
+ TypeInformation<OUT> outTypeInfo = TypeExtractor.getForObject(data.iterator().next());
+ SourceFunction<OUT> function = new FromElementsFunction<OUT>(data);
+
+ return addSource(function, outTypeInfo, "Collection Source");
+ }
+
+ /**
+ * Creates a new DataStream that contains the strings received infinitely
+ * from socket. Received strings are decoded by the system's default
+ * character set.
+ *
+ * @param hostname
+ * The host name which a server socket bind.
+ * @param port
+ * The port number which a server socket bind. A port number of 0
+ * means that the port number is automatically allocated.
+ * @param delimiter
+ * A character which split received strings into records.
+ * @return A DataStream, containing the strings received from socket.
+ */
+ public DataStreamSource<String> socketTextStream(String hostname, int port, char delimiter) {
+ return addSource(new SocketTextStreamFunction(hostname, port, delimiter), null,
+ "Socket Stream");
+ }
+
+ /**
+ * Creates a new DataStream that contains the strings received infinitely
+ * from socket. Received strings are decoded by the system's default
+ * character set, uses '\n' as delimiter.
+ *
+ * @param hostname
+ * The host name which a server socket bind.
+ * @param port
+ * The port number which a server socket bind. A port number of 0
+ * means that the port number is automatically allocated.
+ * @return A DataStream, containing the strings received from socket.
+ */
+ public DataStreamSource<String> socketTextStream(String hostname, int port) {
+ return socketTextStream(hostname, port, '\n');
+ }
+
+ /**
+ * Creates a new DataStream that contains a sequence of numbers.
+ *
+ * @param from
+ * The number to start at (inclusive).
+ * @param to
+ * The number to stop at (inclusive)
+ * @return A DataStrean, containing all number in the [from, to] interval.
+ */
+ public DataStreamSource<Long> generateSequence(long from, long to) {
+ if (from > to) {
+ throw new IllegalArgumentException("Start of sequence must not be greater than the end");
+ }
+ return addSource(new GenSequenceFunction(from, to), null, "Sequence Source");
+ }
+
+ private DataStreamSource<String> addFileSource(InputFormat<String, ?> inputFormat,
+ TypeInformation<String> typeInfo) {
+ FileSourceFunction function = new FileSourceFunction(inputFormat, typeInfo);
+ DataStreamSource<String> returnStream = addSource(function, null, "File Source");
+ streamGraph.setInputFormat(returnStream.getId(), inputFormat);
+ return returnStream;
+ }
+
+ /**
+ * Create a DataStream using a user defined source function for arbitrary
+ * source functionality.</p> By default sources have a parallelism of 1. To
+ * enable parallel execution, the user defined source should implement
+ * {@link ParallelSourceFunction} or extend
+ * {@link RichParallelSourceFunction}. In these cases the resulting source
+ * will have the parallelism of the environment. To change this afterwards
+ * call {@link DataStreamSource#setParallelism(int)}
+ *
+ *
+ * @param function
+ * the user defined function
+ * @param <OUT>
+ * type of the returned stream
+ * @return the data stream constructed
+ */
+ public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function) {
+ return addSource(function, null);
+ }
+
+ /**
+ * Ads a data source with a custom type information thus opening a
+ * {@link DataStream}. Only in very special cases does the user need to
+ * support type information. Otherwise use
+ * {@link #addSource(SourceFunction)} </p> By default sources have a
+ * parallelism of 1. To enable parallel execution, the user defined source
+ * should implement {@link ParallelSourceFunction} or extend
+ * {@link RichParallelSourceFunction}. In these cases the resulting source
+ * will have the parallelism of the environment. To change this afterwards
+ * call {@link DataStreamSource#setParallelism(int)}
+ *
+ * @param function
+ * the user defined function
+ * @param outTypeInfo
+ * the user defined type information for the stream
+ * @param <OUT>
+ * type of the returned stream
+ * @return the data stream constructed
+ */
+ public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function,
+ TypeInformation<OUT> outTypeInfo) {
+ return addSource(function, outTypeInfo, "Custom Source");
+ }
+
+ /**
+ * Ads a data source with a custom type information thus opening a
+ * {@link DataStream}. Only in very special cases does the user need to
+ * support type information. Otherwise use
+ * {@link #addSource(SourceFunction)}
+ *
+ * @param function
+ * the user defined function
+ * @param outTypeInfo
+ * the user defined type information for the stream
+ * @param sourceName
+ * Name of the data source
+ * @param <OUT>
+ * type of the returned stream
+ * @return the data stream constructed
+ */
+ @SuppressWarnings("unchecked")
+ private <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function,
+ TypeInformation<OUT> outTypeInfo, String sourceName) {
+
+ if (outTypeInfo == null) {
+ if (function instanceof GenericSourceFunction) {
+ outTypeInfo = ((GenericSourceFunction<OUT>) function).getType();
+ } else {
+ outTypeInfo = TypeExtractor.createTypeInfo(SourceFunction.class,
+ function.getClass(), 0, null, null);
+ }
+ }
+
+ boolean isParallel = function instanceof ParallelSourceFunction;
+ int dop = isParallel ? getDegreeOfParallelism() : 1;
+
+ StreamInvokable<OUT, OUT> sourceInvokable = new SourceInvokable<OUT>(function);
+
+ DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, sourceName,
+ outTypeInfo, sourceInvokable, isParallel);
+
+ streamGraph.addSourceVertex(returnStream.getId(), sourceInvokable, null, outTypeInfo,
+ sourceName, dop);
+
+ return returnStream;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Instantiation of Execution Contexts
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Creates an execution environment that represents the context in which the
+ * program is currently executed. If the program is invoked standalone, this
+ * method returns a local execution environment, as returned by
+ * {@link #createLocalEnvironment()}.
+ *
+ * @return The execution environment of the context in which the program is
+ * executed.
+ */
+ public static StreamExecutionEnvironment getExecutionEnvironment() {
+ if (currentEnvironment != null) {
+ return currentEnvironment;
+ }
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ if (env instanceof ContextEnvironment) {
+ ContextEnvironment ctx = (ContextEnvironment) env;
+ currentEnvironment = createContextEnvironment(ctx.getClient(), ctx.getJars(),
+ ctx.getDegreeOfParallelism());
+ } else if (env instanceof OptimizerPlanEnvironment | env instanceof PreviewPlanEnvironment) {
+ currentEnvironment = new StreamPlanEnvironment(env);
+ } else {
+ return createLocalEnvironment();
+ }
+ return currentEnvironment;
+ }
+
+ private static StreamExecutionEnvironment createContextEnvironment(Client client,
+ List<File> jars, int dop) {
+ return new StreamContextEnvironment(client, jars, dop);
+ }
+
+ /**
+ * Creates a {@link LocalStreamEnvironment}. The local execution environment
+ * will run the program in a multi-threaded fashion in the same JVM as the
+ * environment was created in. The default degree of parallelism of the
+ * local environment is the number of hardware contexts (CPU cores /
+ * threads), unless it was specified differently by
+ * {@link #setDegreeOfParallelism(int)}.
+ *
+ * @return A local execution environment.
+ */
+ public static LocalStreamEnvironment createLocalEnvironment() {
+ return createLocalEnvironment(defaultLocalDop);
+ }
+
+ /**
+ * Creates a {@link LocalStreamEnvironment}. The local execution environment
+ * will run the program in a multi-threaded fashion in the same JVM as the
+ * environment was created in. It will use the degree of parallelism
+ * specified in the parameter.
+ *
+ * @param degreeOfParallelism
+ * The degree of parallelism for the local environment.
+ * @return A local execution environment with the specified degree of
+ * parallelism.
+ */
+ public static LocalStreamEnvironment createLocalEnvironment(int degreeOfParallelism) {
+ currentEnvironment = new LocalStreamEnvironment();
+ currentEnvironment.setDegreeOfParallelism(degreeOfParallelism);
+ return (LocalStreamEnvironment) currentEnvironment;
+ }
+
+ // TODO:fix cluster default parallelism
+ /**
+ * Creates a {@link RemoteStreamEnvironment}. The remote environment sends
+ * (parts of) the program to a cluster for execution. Note that all file
+ * paths used in the program must be accessible from the cluster. The
+ * execution will use no parallelism, unless the parallelism is set
+ * explicitly via {@link #setDegreeOfParallelism}.
+ *
+ * @param host
+ * The host name or address of the master (JobManager), where the
+ * program should be executed.
+ * @param port
+ * The port of the master (JobManager), where the program should
+ * be executed.
+ * @param jarFiles
+ * The JAR files with code that needs to be shipped to the
+ * cluster. If the program uses user-defined functions,
+ * user-defined input formats, or any libraries, those must be
+ * provided in the JAR files.
+ * @return A remote environment that executes the program on a cluster.
+ */
+ public static StreamExecutionEnvironment createRemoteEnvironment(String host, int port,
+ String... jarFiles) {
+ currentEnvironment = new RemoteStreamEnvironment(host, port, jarFiles);
+ return currentEnvironment;
+ }
+
+ /**
+ * Creates a {@link RemoteStreamEnvironment}. The remote environment sends
+ * (parts of) the program to a cluster for execution. Note that all file
+ * paths used in the program must be accessible from the cluster. The
+ * execution will use the specified degree of parallelism.
+ *
+ * @param host
+ * The host name or address of the master (JobManager), where the
+ * program should be executed.
+ * @param port
+ * The port of the master (JobManager), where the program should
+ * be executed.
+ * @param degreeOfParallelism
+ * The degree of parallelism to use during the execution.
+ * @param jarFiles
+ * The JAR files with code that needs to be shipped to the
+ * cluster. If the program uses user-defined functions,
+ * user-defined input formats, or any libraries, those must be
+ * provided in the JAR files.
+ * @return A remote environment that executes the program on a cluster.
+ */
+ public static StreamExecutionEnvironment createRemoteEnvironment(String host, int port,
+ int degreeOfParallelism, String... jarFiles) {
+ currentEnvironment = new RemoteStreamEnvironment(host, port, jarFiles);
+ currentEnvironment.setDegreeOfParallelism(degreeOfParallelism);
+ return currentEnvironment;
+ }
+
+ /**
+ * Triggers the program execution. The environment will execute all parts of
+ * the program that have resulted in a "sink" operation. Sink operations are
+ * for example printing results or forwarding them to a message queue.
+ * <p>
+ * The program execution will be logged and displayed with a generated
+ * default name.
+ *
+ * @throws Exception
+ **/
+ public abstract void execute() throws Exception;
+
+ /**
+ * Triggers the program execution. The environment will execute all parts of
+ * the program that have resulted in a "sink" operation. Sink operations are
+ * for example printing results or forwarding them to a message queue.
+ * <p>
+ * The program execution will be logged and displayed with the provided name
+ *
+ * @param jobName
+ * Desired name of the job
+ *
+ * @throws Exception
+ **/
+ public abstract void execute(String jobName) throws Exception;
+
+ /**
+ * Getter of the {@link StreamGraph} of the streaming job.
+ *
+ * @return The streamgraph representing the transformations
+ */
+ public StreamGraph getStreamGraph() {
+ return streamGraph;
+ }
+
+ /**
+ * Creates the plan with which the system will execute the program, and
+ * returns it as a String using a JSON representation of the execution data
+ * flow graph. Note that this needs to be called, before the plan is
+ * executed.
+ *
+ * @return The execution plan of the program, as a JSON String.
+ */
+ public String getExecutionPlan() {
+ return getStreamGraph().getStreamingPlanAsJSON();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java
new file mode 100644
index 0000000..1cff7e7
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.environment;
+
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.client.program.Client;
+import org.apache.flink.client.program.Client.OptimizerPlanEnvironment;
+import org.apache.flink.client.program.PackagedProgram.PreviewPlanEnvironment;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.GlobalConfiguration;
+
+public class StreamPlanEnvironment extends StreamExecutionEnvironment {
+
+ private ExecutionEnvironment env;
+
+ protected StreamPlanEnvironment(ExecutionEnvironment env) {
+ super();
+ this.env = env;
+
+ int dop = env.getDegreeOfParallelism();
+ if (dop > 0) {
+ setDegreeOfParallelism(dop);
+ } else {
+ setDegreeOfParallelism(GlobalConfiguration.getInteger(
+ ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE_KEY,
+ ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE));
+ }
+ }
+
+ @Override
+ public void execute() throws Exception {
+ execute("");
+ }
+
+ @Override
+ public void execute(String jobName) throws Exception {
+ currentEnvironment = null;
+
+ streamGraph.setJobName(jobName);
+
+ if (env instanceof OptimizerPlanEnvironment) {
+ ((OptimizerPlanEnvironment) env).setPlan(streamGraph);
+ } else if (env instanceof PreviewPlanEnvironment) {
+ ((PreviewPlanEnvironment) env).setPreview(streamGraph.getStreamingPlanAsJSON());
+ }
+
+ throw new Client.ProgramAbortException();
+ }
+}
[19/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelTranslationTest.java b/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelTranslationTest.java
new file mode 100644
index 0000000..b31618c
--- /dev/null
+++ b/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelTranslationTest.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.spargel.java;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.apache.flink.api.common.aggregators.LongSumAggregator;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.operators.DeltaIteration;
+import org.apache.flink.api.java.operators.DeltaIterationResultSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.TwoInputUdfOperator;
+import org.apache.flink.api.java.tuple.Tuple2;
+
+@SuppressWarnings("serial")
+public class SpargelTranslationTest {
+
+ @Test
+ public void testTranslationPlainEdges() {
+ try {
+ final String ITERATION_NAME = "Test Name";
+
+ final String AGGREGATOR_NAME = "AggregatorName";
+
+ final String BC_SET_MESSAGES_NAME = "borat messages";
+
+ final String BC_SET_UPDATES_NAME = "borat updates";
+ ;
+ final int NUM_ITERATIONS = 13;
+
+ final int ITERATION_DOP = 77;
+
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> bcMessaging = env.fromElements(1L);
+ DataSet<Long> bcUpdate = env.fromElements(1L);
+
+ DataSet<Tuple2<String, Double>> result;
+
+ // ------------ construct the test program ------------------
+ {
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<String, Double>> initialVertices = env.fromElements(new Tuple2<String, Double>("abc", 3.44));
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<String, String>> edges = env.fromElements(new Tuple2<String, String>("a", "c"));
+
+
+ VertexCentricIteration<String, Double, Long, ?> vertexIteration =
+ VertexCentricIteration.withPlainEdges(edges, new UpdateFunction(), new MessageFunctionNoEdgeValue(), NUM_ITERATIONS);
+ vertexIteration.addBroadcastSetForMessagingFunction(BC_SET_MESSAGES_NAME, bcMessaging);
+ vertexIteration.addBroadcastSetForUpdateFunction(BC_SET_UPDATES_NAME, bcUpdate);
+
+ vertexIteration.setName(ITERATION_NAME);
+ vertexIteration.setParallelism(ITERATION_DOP);
+
+ vertexIteration.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());
+
+ result = initialVertices.runOperation(vertexIteration);
+ }
+
+
+ // ------------- validate the java program ----------------
+
+ assertTrue(result instanceof DeltaIterationResultSet);
+
+ DeltaIterationResultSet<?, ?> resultSet = (DeltaIterationResultSet<?, ?>) result;
+ DeltaIteration<?, ?> iteration = (DeltaIteration<?, ?>) resultSet.getIterationHead();
+
+ // check the basic iteration properties
+ assertEquals(NUM_ITERATIONS, resultSet.getMaxIterations());
+ assertArrayEquals(new int[] {0}, resultSet.getKeyPositions());
+ assertEquals(ITERATION_DOP, iteration.getParallelism());
+ assertEquals(ITERATION_NAME, iteration.getName());
+
+ assertEquals(AGGREGATOR_NAME, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());
+
+ // validate that the semantic properties are set as they should
+ TwoInputUdfOperator<?, ?, ?, ?> solutionSetJoin = (TwoInputUdfOperator<?, ?, ?, ?>) resultSet.getNextWorkset();
+ assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(0, 0).contains(0));
+ assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(1, 0).contains(0));
+
+ TwoInputUdfOperator<?, ?, ?, ?> edgesJoin = (TwoInputUdfOperator<?, ?, ?, ?>) solutionSetJoin.getInput1();
+
+ // validate that the broadcast sets are forwarded
+ assertEquals(bcUpdate, solutionSetJoin.getBroadcastSets().get(BC_SET_UPDATES_NAME));
+ assertEquals(bcMessaging, edgesJoin.getBroadcastSets().get(BC_SET_MESSAGES_NAME));
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testTranslationPlainEdgesWithForkedBroadcastVariable() {
+ try {
+ final String ITERATION_NAME = "Test Name";
+
+ final String AGGREGATOR_NAME = "AggregatorName";
+
+ final String BC_SET_MESSAGES_NAME = "borat messages";
+
+ final String BC_SET_UPDATES_NAME = "borat updates";
+ ;
+ final int NUM_ITERATIONS = 13;
+
+ final int ITERATION_DOP = 77;
+
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> bcVar = env.fromElements(1L);
+
+ DataSet<Tuple2<String, Double>> result;
+
+ // ------------ construct the test program ------------------
+ {
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<String, Double>> initialVertices = env.fromElements(new Tuple2<String, Double>("abc", 3.44));
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<String, String>> edges = env.fromElements(new Tuple2<String, String>("a", "c"));
+
+
+ VertexCentricIteration<String, Double, Long, ?> vertexIteration =
+ VertexCentricIteration.withPlainEdges(edges, new UpdateFunction(), new MessageFunctionNoEdgeValue(), NUM_ITERATIONS);
+ vertexIteration.addBroadcastSetForMessagingFunction(BC_SET_MESSAGES_NAME, bcVar);
+ vertexIteration.addBroadcastSetForUpdateFunction(BC_SET_UPDATES_NAME, bcVar);
+
+ vertexIteration.setName(ITERATION_NAME);
+ vertexIteration.setParallelism(ITERATION_DOP);
+
+ vertexIteration.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());
+
+ result = initialVertices.runOperation(vertexIteration);
+ }
+
+
+ // ------------- validate the java program ----------------
+
+ assertTrue(result instanceof DeltaIterationResultSet);
+
+ DeltaIterationResultSet<?, ?> resultSet = (DeltaIterationResultSet<?, ?>) result;
+ DeltaIteration<?, ?> iteration = (DeltaIteration<?, ?>) resultSet.getIterationHead();
+
+ // check the basic iteration properties
+ assertEquals(NUM_ITERATIONS, resultSet.getMaxIterations());
+ assertArrayEquals(new int[] {0}, resultSet.getKeyPositions());
+ assertEquals(ITERATION_DOP, iteration.getParallelism());
+ assertEquals(ITERATION_NAME, iteration.getName());
+
+ assertEquals(AGGREGATOR_NAME, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());
+
+ // validate that the semantic properties are set as they should
+ TwoInputUdfOperator<?, ?, ?, ?> solutionSetJoin = (TwoInputUdfOperator<?, ?, ?, ?>) resultSet.getNextWorkset();
+ assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(0, 0).contains(0));
+ assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(1, 0).contains(0));
+
+ TwoInputUdfOperator<?, ?, ?, ?> edgesJoin = (TwoInputUdfOperator<?, ?, ?, ?>) solutionSetJoin.getInput1();
+
+ // validate that the broadcast sets are forwarded
+ assertEquals(bcVar, solutionSetJoin.getBroadcastSets().get(BC_SET_UPDATES_NAME));
+ assertEquals(bcVar, edgesJoin.getBroadcastSets().get(BC_SET_MESSAGES_NAME));
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static class UpdateFunction extends VertexUpdateFunction<String, Double, Long> {
+
+ @Override
+ public void updateVertex(String vertexKey, Double vertexValue, MessageIterator<Long> inMessages) {}
+ }
+
+ public static class MessageFunctionNoEdgeValue extends MessagingFunction<String, Double, Long, Object> {
+
+ @Override
+ public void sendMessages(String vertexKey, Double vertexValue) {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java b/flink-staging/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java
new file mode 100644
index 0000000..166f7a2
--- /dev/null
+++ b/flink-staging/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.test.spargel;
+
+import java.io.BufferedReader;
+
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.spargel.java.VertexCentricIteration;
+import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.CCMessager;
+import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.CCUpdater;
+import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.IdAssigner;
+import org.apache.flink.test.testdata.ConnectedComponentsData;
+import org.apache.flink.test.util.JavaProgramTestBase;
+
+@SuppressWarnings("serial")
+public class SpargelConnectedComponentsITCase extends JavaProgramTestBase {
+
+ private static final long SEED = 9487520347802987L;
+
+ private static final int NUM_VERTICES = 1000;
+
+ private static final int NUM_EDGES = 10000;
+
+ private String resultPath;
+
+
+ @Override
+ protected void preSubmit() throws Exception {
+ resultPath = getTempFilePath("results");
+ }
+
+ @Override
+ protected void testProgram() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> vertexIds = env.generateSequence(1, NUM_VERTICES);
+ DataSet<String> edgeString = env.fromElements(ConnectedComponentsData.getRandomOddEvenEdges(NUM_EDGES, NUM_VERTICES, SEED).split("\n"));
+
+ DataSet<Tuple2<Long, Long>> edges = edgeString.map(new EdgeParser());
+
+ DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
+ DataSet<Tuple2<Long, Long>> result = initialVertices.runOperation(VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100));
+
+ result.writeAsCsv(resultPath, "\n", " ");
+ env.execute("Spargel Connected Components");
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ for (BufferedReader reader : getResultReader(resultPath)) {
+ ConnectedComponentsData.checkOddEvenResult(reader);
+ }
+ }
+
+ public static final class EdgeParser extends RichMapFunction<String, Tuple2<Long, Long>> {
+ public Tuple2<Long, Long> map(String value) {
+ String[] nums = value.split(" ");
+ return new Tuple2<Long, Long>(Long.parseLong(nums[0]), Long.parseLong(nums[1]));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/test/resources/log4j-test.properties b/flink-staging/flink-spargel/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..2fb9345
--- /dev/null
+++ b/flink-staging/flink-spargel/src/test/resources/log4j-test.properties
@@ -0,0 +1,19 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/test/resources/logback-test.xml b/flink-staging/flink-spargel/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..8b3bb27
--- /dev/null
+++ b/flink-staging/flink-spargel/src/test/resources/logback-test.xml
@@ -0,0 +1,29 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml
new file mode 100644
index 0000000..df34b20
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml
@@ -0,0 +1,243 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-streaming-connectors</artifactId>
+ <name>flink-streaming-connectors</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ <version>0.8.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.sf.jopt-simple</groupId>
+ <artifactId>jopt-simple</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-reflect</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.yammer.metrics</groupId>
+ <artifactId>metrics-annotation</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.rabbitmq</groupId>
+ <artifactId>amqp-client</artifactId>
+ <version>3.3.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-core</artifactId>
+ <version>1.5.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.tukaani</groupId>
+ <artifactId>xz</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>hbc-core</artifactId>
+ <version>2.2.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ <version>1.8</version>
+ </dependency>
+
+ <dependency>
+ <groupId>redis.clients</groupId>
+ <artifactId>jedis</artifactId>
+ <version>2.4.2</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.simple-spring-memcached</groupId>
+ <artifactId>spymemcached</artifactId>
+ <version>2.8.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.sling</groupId>
+ <artifactId>org.apache.sling.commons.json</artifactId>
+ <version>2.0.6</version>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
new file mode 100644
index 0000000..1623943
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.function.source.GenericSourceFunction;
+import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.connectors.util.DeserializationSchema;
+
+public abstract class ConnectorSource<OUT> extends RichParallelSourceFunction<OUT> implements
+ GenericSourceFunction<OUT> {
+
+ private static final long serialVersionUID = 1L;
+ protected DeserializationSchema<OUT> schema;
+
+ public ConnectorSource(DeserializationSchema<OUT> schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public TypeInformation<OUT> getType() {
+ return TypeExtractor.createTypeInfo(DeserializationSchema.class, schema.getClass(), 0,
+ null, null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/CustomSerializationDBState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/CustomSerializationDBState.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/CustomSerializationDBState.java
new file mode 100644
index 0000000..9b1de36
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/CustomSerializationDBState.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.db;
+
+import java.io.Serializable;
+
+public abstract class CustomSerializationDBState<K extends Serializable, V extends Serializable> {
+
+ protected DBSerializer<K> keySerializer;
+ protected DBSerializer<V> valueSerializer;
+
+ public CustomSerializationDBState(DBSerializer<K> keySerializer, DBSerializer<V> valueSerializer) {
+ this.keySerializer = keySerializer;
+ this.valueSerializer = valueSerializer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBSerializer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBSerializer.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBSerializer.java
new file mode 100644
index 0000000..a80e20c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBSerializer.java
@@ -0,0 +1,36 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.flink.streaming.connectors.db;
+
+import java.io.Serializable;
+
+/**
+ * Interface for custom serialization of keys and values used in external
+ * databases.
+ *
+ * @param <T>
+ * Type of the key or value to serialize
+ */
+public interface DBSerializer<T extends Serializable> {
+
+ byte[] write(T object);
+
+ T read(byte[] serializedObject);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBState.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBState.java
new file mode 100644
index 0000000..7528182
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBState.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.db;
+
+public interface DBState<K, V> {
+
+ public void put(K key, V value);
+
+ public V get(K key);
+
+ public void remove(K key);
+
+ public void close();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateIterator.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateIterator.java
new file mode 100644
index 0000000..890d933
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateIterator.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.db;
+
+import java.io.Serializable;
+
+public abstract class DBStateIterator<K extends Serializable, V extends Serializable> {
+
+ public abstract boolean hasNext();
+
+ public abstract K getNextKey();
+
+ public abstract V getNextValue();
+
+ public abstract void next();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateWithIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateWithIterator.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateWithIterator.java
new file mode 100644
index 0000000..3383330
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateWithIterator.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.db;
+
+import java.io.Serializable;
+
+public interface DBStateWithIterator<K extends Serializable, V extends Serializable> extends DBState<K, V> {
+
+ public DBStateIterator<K, V> getIterator();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DefaultDBSerializer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DefaultDBSerializer.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DefaultDBSerializer.java
new file mode 100644
index 0000000..6b770cb
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DefaultDBSerializer.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.db;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang3.SerializationUtils;
+
+public class DefaultDBSerializer<T extends Serializable> implements DBSerializer<T> {
+
+ @Override
+ public byte[] write(T object) {
+ return SerializationUtils.serialize(object);
+ }
+
+ @Override
+ public T read(byte[] serializedObject) {
+ return SerializationUtils.deserialize(serializedObject);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/LevelDBState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/LevelDBState.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/LevelDBState.java
new file mode 100644
index 0000000..2e8626d
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/LevelDBState.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.db;
+
+import static org.fusesource.leveldbjni.JniDBFactory.factory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.iq80.leveldb.DB;
+import org.iq80.leveldb.DBIterator;
+import org.iq80.leveldb.Options;
+
+/**
+ * Interface to a LevelDB key-value store.
+ *
+ * @see <a href="https://code.google.com/p/leveldb/">https://code.google.com/p/leveldb/</a>
+ *
+ * @param <K>
+ * Type of key
+ * @param <V>
+ * Type of value
+ */
+public class LevelDBState<K extends Serializable, V extends Serializable> extends
+ CustomSerializationDBState<K, V> implements DBStateWithIterator<K, V> {
+
+ private DB database;
+
+ public LevelDBState(String dbName, DBSerializer<K> keySerializer,
+ DBSerializer<V> valueSerializer) {
+ super(keySerializer, valueSerializer);
+ Options options = new Options();
+ File file = new File(dbName);
+ options.createIfMissing(true);
+ try {
+ factory.destroy(file, options);
+ database = factory.open(file, options);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public LevelDBState(String dbName) {
+ this(dbName, new DefaultDBSerializer<K>(), new DefaultDBSerializer<V>());
+ }
+
+ @Override
+ public void close() {
+ try {
+ database.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void put(K key, V value) {
+ database.put(keySerializer.write(key), valueSerializer.write(value));
+ }
+
+ @Override
+ public V get(K key) {
+ byte[] serializedValue = database.get(keySerializer.write(key));
+ if (serializedValue != null) {
+ return valueSerializer.read(serializedValue);
+ } else {
+ throw new RuntimeException("No such entry at key " + key);
+ }
+ }
+
+ @Override
+ public void remove(K key) {
+ database.delete(keySerializer.write(key));
+ }
+
+ @Override
+ public DBStateIterator<K, V> getIterator() {
+ return new LevelDBStateIterator();
+ }
+
+ private class LevelDBStateIterator extends DBStateIterator<K, V> {
+ private DBIterator iterator;
+
+ public LevelDBStateIterator() {
+ this.iterator = database.iterator();
+ this.iterator.seekToFirst();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public K getNextKey() {
+ return keySerializer.read(iterator.peekNext().getKey());
+ }
+
+ @Override
+ public V getNextValue() {
+ return valueSerializer.read(iterator.peekNext().getValue());
+ }
+
+ @Override
+ public void next() {
+ iterator.next();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/MemcachedState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/MemcachedState.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/MemcachedState.java
new file mode 100644
index 0000000..7f06a1f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/MemcachedState.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.db;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import net.spy.memcached.MemcachedClient;
+
+/**
+ * Interface to a Memcached key-value cache. It needs a running instance of Memcached.
+ *
+ * @see <a href="http://memcached.org/">http://memcached.org</a>
+ *
+ * @param <V>
+ * Type of value
+ */
+public class MemcachedState<V> implements DBState<String, V> {
+
+ private MemcachedClient memcached;
+
+ public MemcachedState() {
+ try {
+ memcached = new MemcachedClient(new InetSocketAddress("localhost", 11211));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public MemcachedState(String hostname, int portNum) {
+ try {
+ memcached = new MemcachedClient(new InetSocketAddress(hostname, portNum));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() {
+ memcached.shutdown();
+ }
+
+ @Override
+ public void put(String key, V value) {
+ memcached.set(key, 0, value);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public V get(String key) {
+ return (V) memcached.get(key);
+ }
+
+ @Override
+ public void remove(String key) {
+ memcached.delete(key.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/RedisState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/RedisState.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/RedisState.java
new file mode 100644
index 0000000..b9244c5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/RedisState.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.db;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.Set;
+
+import redis.clients.jedis.Jedis;
+
+/**
+ * Interface to a Redis key-value cache. It needs a running instance of Redis.
+ *
+ * @see <a href="http://redis.io/">http://redis.io/</a>
+ *
+ * @param <K>
+ * Type of key
+ * @param <V>
+ * Type of value
+ */
+public class RedisState<K extends Serializable, V extends Serializable> extends
+ CustomSerializationDBState<K, V> implements DBStateWithIterator<K, V> {
+
+ private Jedis jedis;
+
+ public RedisState(DBSerializer<K> keySerializer, DBSerializer<V> valueSerializer) {
+ super(keySerializer, valueSerializer);
+ jedis = new Jedis("localhost");
+ }
+
+ public RedisState() {
+ this(new DefaultDBSerializer<K>(), new DefaultDBSerializer<V>());
+ }
+
+ @Override
+ public void close() {
+ jedis.close();
+ }
+
+ @Override
+ public void put(K key, V value) {
+ jedis.set(keySerializer.write(key), valueSerializer.write(value));
+ }
+
+ @Override
+ public V get(K key) {
+ return valueSerializer.read(jedis.get(keySerializer.write(key)));
+ }
+
+ @Override
+ public void remove(K key) {
+ jedis.del(keySerializer.write(key));
+ }
+
+ @Override
+ public DBStateIterator<K, V> getIterator() {
+ return new RedisStateIterator();
+ }
+
+ private class RedisStateIterator extends DBStateIterator<K, V> {
+
+ private Set<byte[]> set;
+ private Iterator<byte[]> iterator;
+ private byte[] currentKey;
+
+ public RedisStateIterator() {
+ set = jedis.keys(new byte[0]);
+ jedis.keys("*".getBytes()).iterator();
+ iterator = set.iterator();
+ currentKey = iterator.next();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public K getNextKey() {
+ return keySerializer.read(currentKey);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public V getNextValue() {
+ return (V) jedis.get(currentKey);
+ }
+
+ @Override
+ public void next() {
+ currentKey = iterator.next();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
new file mode 100644
index 0000000..8a2f2b8
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.flume;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
+import org.apache.flink.streaming.connectors.util.SerializationSchema;
+import org.apache.flume.Event;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.FlumeException;
+import org.apache.flume.api.RpcClient;
+import org.apache.flume.api.RpcClientFactory;
+import org.apache.flume.event.EventBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FlumeSink<IN> extends RichSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(FlumeSink.class);
+
+ private transient FlinkRpcClientFacade client;
+ boolean initDone = false;
+ String host;
+ int port;
+ SerializationSchema<IN, byte[]> scheme;
+
+ public FlumeSink(String host, int port, SerializationSchema<IN, byte[]> schema) {
+ this.host = host;
+ this.port = port;
+ this.scheme = schema;
+ }
+
+ /**
+ * Receives tuples from the Apache Flink {@link DataStream} and forwards
+ * them to Apache Flume.
+ *
+ * @param value
+ * The tuple arriving from the datastream
+ */
+ @Override
+ public void invoke(IN value) {
+
+ byte[] data = scheme.serialize(value);
+ client.sendDataToFlume(data);
+
+ }
+
+ private class FlinkRpcClientFacade {
+ private RpcClient client;
+ private String hostname;
+ private int port;
+
+ /**
+ * Initializes the connection to Apache Flume.
+ *
+ * @param hostname
+ * The host
+ * @param port
+ * The port.
+ */
+ public void init(String hostname, int port) {
+ // Setup the RPC connection
+ this.hostname = hostname;
+ this.port = port;
+ int initCounter = 0;
+ while (true) {
+ if (initCounter >= 90) {
+ throw new RuntimeException("Cannot establish connection with" + port + " at "
+ + host);
+ }
+ try {
+ this.client = RpcClientFactory.getDefaultInstance(hostname, port);
+ } catch (FlumeException e) {
+ // Wait one second if the connection failed before the next
+ // try
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e1) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Interrupted while trying to connect {} at {}", port, host);
+ }
+ }
+ }
+ if (client != null) {
+ break;
+ }
+ initCounter++;
+ }
+ initDone = true;
+ }
+
+ /**
+ * Sends byte arrays as {@link Event} series to Apache Flume.
+ *
+ * @param data
+ * The byte array to send to Apache FLume
+ */
+ public void sendDataToFlume(byte[] data) {
+ Event event = EventBuilder.withBody(data);
+
+ try {
+ client.append(event);
+
+ } catch (EventDeliveryException e) {
+ // clean up and recreate the client
+ client.close();
+ client = null;
+ client = RpcClientFactory.getDefaultInstance(hostname, port);
+ }
+ }
+
+ }
+
+ @Override
+ public void close() {
+ client.client.close();
+ }
+
+ @Override
+ public void open(Configuration config) {
+ client = new FlinkRpcClientFacade();
+ client.init(host, port);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
new file mode 100644
index 0000000..4f6ec2d
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.flume;
+
+import java.util.List;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.connectors.ConnectorSource;
+import org.apache.flink.streaming.connectors.util.DeserializationSchema;
+import org.apache.flink.util.Collector;
+import org.apache.flume.Context;
+import org.apache.flume.channel.ChannelProcessor;
+import org.apache.flume.source.AvroSource;
+import org.apache.flume.source.avro.AvroFlumeEvent;
+import org.apache.flume.source.avro.Status;
+
+public class FlumeSource<OUT> extends ConnectorSource<OUT> {
+ private static final long serialVersionUID = 1L;
+
+ String host;
+ String port;
+ volatile boolean finished = false;
+
+ FlumeSource(String host, int port, DeserializationSchema<OUT> deserializationSchema) {
+ super(deserializationSchema);
+ this.host = host;
+ this.port = Integer.toString(port);
+ }
+
+ public class MyAvroSource extends AvroSource {
+ Collector<OUT> collector;
+
+ /**
+ * Sends the AvroFlumeEvent from it's argument list to the Apache Flink
+ * {@link DataStream}.
+ *
+ * @param avroEvent
+ * The event that should be sent to the dataStream
+ * @return A {@link Status}.OK message if sending the event was
+ * successful.
+ */
+ @Override
+ public Status append(AvroFlumeEvent avroEvent) {
+ collect(avroEvent);
+ return Status.OK;
+ }
+
+ /**
+ * Sends the AvroFlumeEvents from it's argument list to the Apache Flink
+ * {@link DataStream}.
+ *
+ * @param events
+ * The events that is sent to the dataStream
+ * @return A Status.OK message if sending the events was successful.
+ */
+ @Override
+ public Status appendBatch(List<AvroFlumeEvent> events) {
+ for (AvroFlumeEvent avroEvent : events) {
+ collect(avroEvent);
+ }
+
+ return Status.OK;
+ }
+
+ /**
+ * Deserializes the AvroFlumeEvent before sending it to the Apache Flink
+ * {@link DataStream}.
+ *
+ * @param avroEvent
+ * The event that is sent to the dataStream
+ */
+ private void collect(AvroFlumeEvent avroEvent) {
+ byte[] b = avroEvent.getBody().array();
+ OUT out = FlumeSource.this.schema.deserialize(b);
+
+ if (schema.isEndOfStream(out)) {
+ FlumeSource.this.finished = true;
+ this.stop();
+ FlumeSource.this.notifyAll();
+ } else {
+ collector.collect(out);
+ }
+
+ }
+
+ }
+
+ MyAvroSource avroSource;
+
+ /**
+ * Configures the AvroSource. Also sets the collector so the application can
+ * use it from outside of the invoke function.
+ *
+ * @param collector
+ * The collector used in the invoke function
+ */
+ public void configureAvroSource(Collector<OUT> collector) {
+
+ avroSource = new MyAvroSource();
+ avroSource.collector = collector;
+ Context context = new Context();
+ context.put("port", port);
+ context.put("bind", host);
+ avroSource.configure(context);
+ // An instance of a ChannelProcessor is required for configuring the
+ // avroSource although it will not be used in this case.
+ ChannelProcessor cp = new ChannelProcessor(null);
+ avroSource.setChannelProcessor(cp);
+ }
+
+ /**
+ * Configures the AvroSource and runs until the user calls a close function.
+ *
+ * @param collector
+ * The Collector for sending data to the datastream
+ */
+ @Override
+ public void invoke(Collector<OUT> collector) throws Exception {
+ configureAvroSource(collector);
+ avroSource.start();
+ while (!finished) {
+ this.wait();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
new file mode 100644
index 0000000..3cfd7d4
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.flume;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.util.SerializationSchema;
+import org.apache.flink.streaming.connectors.util.SimpleStringSchema;
+
+public class FlumeTopology {
+
+ public static void main(String[] args) throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+ @SuppressWarnings("unused")
+ DataStream<String> dataStream1 = env.addSource(
+ new FlumeSource<String>("localhost", 41414, new SimpleStringSchema())).addSink(
+ new FlumeSink<String>("localhost", 42424, new StringToByteSerializer()));
+
+ env.execute();
+ }
+
+ public static class StringToByteSerializer implements SerializationSchema<String, byte[]> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public byte[] serialize(String element) {
+ return element.getBytes();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
new file mode 100644
index 0000000..0f16541
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.json;
+
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.sling.commons.json.JSONException;
+
+/**
+ * Abstract class derived from {@link RichFlatMapFunction} to handle JSON files.
+ *
+ * @param <IN>
+ * Type of the input elements.
+ * @param <OUT>
+ * Type of the returned elements.
+ */
+public abstract class JSONParseFlatMap<IN, OUT> extends RichFlatMapFunction<IN, OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ // private static final Log LOG = LogFactory.getLog(JSONParseFlatMap.class);
+
+ /**
+ * Get the value object associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public Object get(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).get("retValue");
+ }
+
+ /**
+ * Get the boolean value associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public boolean getBoolean(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).getBoolean("retValue");
+ }
+
+ /**
+ * Get the double value associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public double getDouble(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).getDouble("retValue");
+ }
+
+ /**
+ * Get the int value associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public int getInt(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).getInt("retValue");
+ }
+
+ /**
+ * Get the long value associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public long getLong(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).getLong("retValue");
+ }
+
+ /**
+ * Get the String value associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public String getString(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).getString("retValue");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
new file mode 100644
index 0000000..c1eabbd
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.json;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.StringTokenizer;
+
+import org.apache.sling.commons.json.JSONArray;
+import org.apache.sling.commons.json.JSONException;
+import org.apache.sling.commons.json.JSONObject;
+
+/**
+ * A JSONParser contains a JSONObject and provides opportunity to access
+ * embedded fields in JSON code.
+ */
+public class JSONParser {
+
+ private JSONObject originalJO;
+ private String searchedfield;
+ private Object temp;
+
+ /**
+ * Construct a JSONParser from a string. The string has to be a JSON code
+ * from which we want to get a field.
+ *
+ * @param jsonText
+ * A string which contains a JSON code. String representation of
+ * a JSON code.
+ * @throws JSONException
+ * If there is a syntax error in the source string.
+ */
+ public JSONParser(String jsonText) throws JSONException {
+ originalJO = new JSONObject(jsonText);
+ }
+
+ /**
+ *
+ * Parse the JSON code passed to the constructor to find the given key.
+ *
+ * @param key
+ * The key whose value is searched for.
+ * @return A JSONObject which has only one field called "retValue" and the
+ * value associated to it is the searched value. The methods of
+ * JSONObject can be used to get the field value in a desired
+ * format.
+ * @throws JSONException
+ * If the key is not found.
+ */
+ public JSONObject parse(String key) throws JSONException {
+ initializeParser(key);
+ parsing();
+ return putResultInJSONObj();
+ }
+
+ /**
+ * Prepare the fields of the class for the parsing
+ *
+ * @param key
+ * The key whose value is searched for.
+ * @throws JSONException
+ * If the key is not found.
+ */
+ private void initializeParser(String key) throws JSONException {
+ searchedfield = key;
+ temp = new JSONObject(originalJO.toString());
+ }
+
+ /**
+ * This function goes through the given field and calls the appropriate
+ * functions to treat the units between the punctuation marks.
+ *
+ * @throws JSONException
+ * If the key is not found.
+ */
+ private void parsing() throws JSONException {
+ StringTokenizer st = new StringTokenizer(searchedfield, ".");
+ while (st.hasMoreTokens()) {
+ find(st.nextToken());
+ }
+ }
+
+ /**
+ * Search for the next part of the field and update the state if it was
+ * found.
+ *
+ * @param nextToken
+ * The current part of the searched field.
+ * @throws JSONException
+ * If the key is not found.
+ */
+ private void find(String nextToken) throws JSONException {
+ if (endsWithBracket(nextToken)) {
+ treatAllBracket(nextToken);
+ } else {
+ temp = ((JSONObject) temp).get(nextToken);
+ }
+ }
+
+ /**
+ * Determine whether the given string ends with a closing square bracket ']'
+ *
+ * @param nextToken
+ * The current part of the searched field.
+ * @return True if the given string ends with a closing square bracket ']'
+ * and false otherwise.
+ */
+ private boolean endsWithBracket(String nextToken) {
+ return nextToken.substring(nextToken.length() - 1).endsWith("]");
+ }
+
+ /**
+ * Handle (multidimensional) arrays. Treat the square bracket pairs one
+ * after the other if necessary.
+ *
+ * @param nextToken
+ * The current part of the searched field.
+ * @throws JSONException
+ * If the searched element is not found.
+ */
+ private void treatAllBracket(String nextToken) throws JSONException {
+ List<String> list = Arrays.asList(nextToken.split("\\["));
+ ListIterator<String> iter = list.listIterator();
+
+ temp = ((JSONObject) temp).get(iter.next());
+
+ while (iter.hasNext()) {
+ int index = Integer.parseInt(cutBracket(iter.next()));
+ temp = ((JSONArray) temp).get(index);
+ }
+ }
+
+ /**
+ * Remove the last character of the string.
+ *
+ * @param string
+ * String to modify.
+ * @return The given string without the last character.
+ */
+ private String cutBracket(String string) {
+ return string.substring(0, string.length() - 1);
+ }
+
+ /**
+ * Save the result of the search into a JSONObject.
+ *
+ * @return A special JSONObject which contain only one key. The value
+ * associated to this key is the result of the search.
+ * @throws JSONException
+ * If there is a problem creating the JSONObject. (e.g. invalid
+ * syntax)
+ */
+ private JSONObject putResultInJSONObj() throws JSONException {
+ JSONObject jo = new JSONObject();
+ jo.put("retValue", temp);
+ return jo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
new file mode 100644
index 0000000..9bb87a0
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.util.Properties;
+
+import kafka.javaapi.producer.Producer;
+import kafka.producer.KeyedMessage;
+import kafka.producer.ProducerConfig;
+
+import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
+import org.apache.flink.streaming.connectors.util.SerializationSchema;
+
+public class KafkaSink<IN, OUT> extends RichSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+
+ private kafka.javaapi.producer.Producer<Integer, OUT> producer;
+ private Properties props;
+ private String topicId;
+ private String brokerAddr;
+ private boolean initDone = false;
+ private SerializationSchema<IN, OUT> scheme;
+
+ public KafkaSink(String topicId, String brokerAddr,
+ SerializationSchema<IN, OUT> serializationSchema) {
+ this.topicId = topicId;
+ this.brokerAddr = brokerAddr;
+ this.scheme = serializationSchema;
+
+ }
+
+ /**
+ * Initializes the connection to Kafka.
+ */
+ public void initialize() {
+ props = new Properties();
+
+ props.put("metadata.broker.list", brokerAddr);
+ props.put("serializer.class", "kafka.serializer.StringEncoder");
+ props.put("request.required.acks", "1");
+
+ ProducerConfig config = new ProducerConfig(props);
+ producer = new Producer<Integer, OUT>(config);
+ initDone = true;
+ }
+
+ /**
+ * Called when new data arrives to the sink, and forwards it to Kafka.
+ *
+ * @param next
+ * The incoming data
+ */
+ @Override
+ public void invoke(IN next) {
+ if (!initDone) {
+ initialize();
+ }
+
+ producer.send(new KeyedMessage<Integer, OUT>(topicId, scheme.serialize(next)));
+
+ }
+
+ @Override
+ public void close() {
+ producer.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
new file mode 100644
index 0000000..7328500
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import kafka.consumer.ConsumerConfig;
+import kafka.consumer.ConsumerIterator;
+import kafka.consumer.KafkaStream;
+import kafka.javaapi.consumer.ConsumerConnector;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.connectors.ConnectorSource;
+import org.apache.flink.streaming.connectors.util.DeserializationSchema;
+import org.apache.flink.util.Collector;
+
+public class KafkaSource<OUT> extends ConnectorSource<OUT> {
+ private static final long serialVersionUID = 1L;
+
+ private final String zkQuorum;
+ private final String groupId;
+ private final String topicId;
+ private ConsumerConnector consumer;
+
+ OUT outTuple;
+
+ public KafkaSource(String zkQuorum, String groupId, String topicId,
+ DeserializationSchema<OUT> deserializationSchema) {
+ super(deserializationSchema);
+ this.zkQuorum = zkQuorum;
+ this.groupId = groupId;
+ this.topicId = topicId;
+ }
+
+ /**
+ * Initializes the connection to Kafka.
+ */
+ private void initializeConnection() {
+ Properties props = new Properties();
+ props.put("zookeeper.connect", zkQuorum);
+ props.put("group.id", groupId);
+ props.put("zookeeper.session.timeout.ms", "2000");
+ props.put("zookeeper.sync.time.ms", "200");
+ props.put("auto.commit.interval.ms", "1000");
+ consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(props));
+ }
+
+ /**
+ * Called to forward the data from the source to the {@link DataStream}.
+ *
+ * @param collector
+ * The Collector for sending data to the dataStream
+ */
+ @Override
+ public void invoke(Collector<OUT> collector) throws Exception {
+
+ Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer
+ .createMessageStreams(Collections.singletonMap(topicId, 1));
+
+ KafkaStream<byte[], byte[]> stream = consumerMap.get(topicId).get(0);
+ ConsumerIterator<byte[], byte[]> it = stream.iterator();
+
+ while (it.hasNext()) {
+ OUT out = schema.deserialize(it.next().message());
+ if (schema.isEndOfStream(out)) {
+ break;
+ }
+ collector.collect(out);
+ }
+ consumer.shutdown();
+ }
+
+ @Override
+ public void open(Configuration config) {
+ initializeConnection();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
new file mode 100644
index 0000000..7801d56
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.connectors.util.SimpleStringSchema;
+import org.apache.flink.util.Collector;
+
+public class KafkaTopology {
+
+ public static final class MySource implements SourceFunction<String> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void invoke(Collector<String> collector) throws Exception {
+ for (int i = 0; i < 10; i++) {
+ collector.collect(new String(Integer.toString(i)));
+ }
+ collector.collect(new String("q"));
+
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings("unused")
+ DataStream<String> stream1 = env
+ .addSource(
+ new KafkaSource<String>("localhost:2181", "group", "test",
+ new SimpleStringSchema())).print();
+
+ @SuppressWarnings("unused")
+ DataStream<String> stream2 = env.addSource(new MySource()).addSink(
+ new KafkaSink<String, String>("test", "localhost:9092", new SimpleStringSchema()));
+
+ env.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
new file mode 100644
index 0000000..38c4f5f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.rabbitmq;
+
+import java.io.IOException;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
+import org.apache.flink.streaming.connectors.util.SerializationSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+
+public class RMQSink<IN> extends RichSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(RMQSink.class);
+
+ private String QUEUE_NAME;
+ private String HOST_NAME;
+ private transient ConnectionFactory factory;
+ private transient Connection connection;
+ private transient Channel channel;
+ private SerializationSchema<IN, byte[]> scheme;
+
+ public RMQSink(String HOST_NAME, String QUEUE_NAME, SerializationSchema<IN, byte[]> schema) {
+ this.HOST_NAME = HOST_NAME;
+ this.QUEUE_NAME = QUEUE_NAME;
+ this.scheme = schema;
+ }
+
+ /**
+ * Initializes the connection to RMQ.
+ */
+ public void initializeConnection() {
+ factory = new ConnectionFactory();
+ factory.setHost(HOST_NAME);
+ try {
+ connection = factory.newConnection();
+ channel = connection.createChannel();
+ channel.queueDeclare(QUEUE_NAME, false, false, false, null);
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Called when new data arrives to the sink, and forwards it to RMQ.
+ *
+ * @param value
+ * The incoming data
+ */
+ @Override
+ public void invoke(IN value) {
+ try {
+ byte[] msg = scheme.serialize(value);
+
+ channel.basicPublish("", QUEUE_NAME, null, msg);
+
+ } catch (IOException e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Cannot send RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
+ }
+ }
+
+ }
+
+ /**
+ * Closes the connection.
+ */
+ private void closeChannel() {
+ try {
+ channel.close();
+ connection.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Error while closing RMQ connection with " + QUEUE_NAME
+ + " at " + HOST_NAME, e);
+ }
+
+ }
+
+ @Override
+ public void open(Configuration config) {
+ initializeConnection();
+ }
+
+ @Override
+ public void close() {
+ closeChannel();
+ }
+
+}
[08/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/MapCheckpoint.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/MapCheckpoint.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/MapCheckpoint.java
new file mode 100644
index 0000000..15d1fd5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/MapCheckpoint.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state.checkpoint;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.streaming.state.MapState;
+import org.apache.flink.streaming.state.OperatorState;
+
+public class MapCheckpoint<K, V> extends StateCheckpoint<Map<K, V>> {
+
+ private static final long serialVersionUID = 1L;
+
+ protected Set<K> removedItems;
+ protected Map<K, V> updatedItems;
+ protected boolean clear;
+
+ @SuppressWarnings("unchecked")
+ public MapCheckpoint(OperatorState<Map<K, V>> operatorState) {
+ if (operatorState instanceof MapState) {
+ MapState<K, V> mapState = (MapState<K, V>) operatorState;
+
+ this.removedItems = mapState.getRemovedItems();
+ this.clear = mapState.isCleared();
+
+ this.updatedItems = new HashMap<K, V>();
+ for (K key : mapState.getUpdatedItems()) {
+ this.updatedItems.put(key, mapState.get(key));
+ }
+ this.checkpointedState = this.updatedItems;
+
+ } else {
+ throw new RuntimeException("MapCheckpoint can only be used with MapState");
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public StateCheckpoint<Map<K, V>> update(StateCheckpoint<Map<K, V>> nextCheckpoint) {
+ MapCheckpoint<K, V> mapCheckpoint = (MapCheckpoint<K, V>) nextCheckpoint;
+ if (this.checkpointedState == null) {
+ this.checkpointedState = mapCheckpoint.updatedItems;
+ } else {
+ if (mapCheckpoint.clear) {
+ this.checkpointedState.clear();
+ }
+ for (Object key : mapCheckpoint.removedItems) {
+ this.checkpointedState.remove(key);
+ }
+ this.checkpointedState.putAll(mapCheckpoint.updatedItems);
+ }
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/StateCheckpoint.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/StateCheckpoint.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/StateCheckpoint.java
new file mode 100644
index 0000000..8b76245
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/StateCheckpoint.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state.checkpoint;
+
+import java.io.Serializable;
+
+import org.apache.flink.streaming.state.OperatorState;
+
+/**
+ * Base class for creating checkpoints for {@link OperatorState}. This
+ * checkpoints will be used to backup states in stateful Flink operators and
+ * also to restore them in case of node failure. To allow incremental
+ * checkpoints override the {@link #update(StateCheckpoint)} method.
+ *
+ * @param <T>
+ * The type of the state.
+ */
+public class StateCheckpoint<T> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ T checkpointedState;
+
+ /**
+ * Creates a state checkpoint from the given {@link OperatorState}
+ *
+ * @param operatorState
+ * The {@link OperatorState} to checkpoint.
+ */
+ public StateCheckpoint(OperatorState<T> operatorState) {
+ this.checkpointedState = operatorState.getState();
+ }
+
+ public StateCheckpoint() {
+ this.checkpointedState = null;
+ }
+
+ /**
+ * Returns the state object for the checkpoint.
+ *
+ * @return The checkpointed state object.
+ */
+ public T getCheckpointedState() {
+ return checkpointedState;
+ }
+
+ /**
+ * Updates the checkpoint from next one. Override this method to allow
+ * incremental updates.
+ *
+ * @param nextCheckpoint
+ * The {@link StateCheckpoint} will be used to update from.
+ */
+ public StateCheckpoint<T> update(StateCheckpoint<T> nextCheckpoint) {
+ this.checkpointedState = nextCheckpoint.getCheckpointedState();
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return checkpointedState.toString();
+ }
+
+ public boolean stateEquals(StateCheckpoint<T> other) {
+ return checkpointedState.equals(other.checkpointedState);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/ClusterUtil.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/ClusterUtil.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/ClusterUtil.java
new file mode 100755
index 0000000..691b111
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/ClusterUtil.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.client.JobClient;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import akka.actor.ActorRef;
+
+public class ClusterUtil {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ClusterUtil.class);
+ public static final String CANNOT_EXECUTE_EMPTY_JOB = "Cannot execute empty job";
+
+ /**
+ * Executes the given JobGraph locally, on a FlinkMiniCluster
+ *
+ * @param jobGraph
+ * jobGraph
+ * @param degreeOfParallelism
+ * numberOfTaskTrackers
+ * @param memorySize
+ * memorySize
+ */
+ public static void runOnMiniCluster(JobGraph jobGraph, int degreeOfParallelism, long memorySize)
+ throws Exception {
+
+ Configuration configuration = jobGraph.getJobConfiguration();
+
+ LocalFlinkMiniCluster exec = null;
+
+ configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, memorySize);
+ configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, degreeOfParallelism);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Running on mini cluster");
+ }
+
+ try {
+ exec = new LocalFlinkMiniCluster(configuration, true);
+ ActorRef jobClient = exec.getJobClient();
+
+ JobClient.submitJobAndWait(jobGraph, true, jobClient, exec.timeout());
+
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ if (exec != null) {
+ exec.stop();
+ }
+ }
+ }
+
+ public static void runOnMiniCluster(JobGraph jobGraph, int numOfSlots) throws Exception {
+ runOnMiniCluster(jobGraph, numOfSlots, -1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
new file mode 100644
index 0000000..73e6360
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util.keys;
+
+import java.lang.reflect.Array;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.common.typeutils.TypeComparator;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.Keys;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple10;
+import org.apache.flink.api.java.tuple.Tuple11;
+import org.apache.flink.api.java.tuple.Tuple12;
+import org.apache.flink.api.java.tuple.Tuple13;
+import org.apache.flink.api.java.tuple.Tuple14;
+import org.apache.flink.api.java.tuple.Tuple15;
+import org.apache.flink.api.java.tuple.Tuple16;
+import org.apache.flink.api.java.tuple.Tuple17;
+import org.apache.flink.api.java.tuple.Tuple18;
+import org.apache.flink.api.java.tuple.Tuple19;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple20;
+import org.apache.flink.api.java.tuple.Tuple21;
+import org.apache.flink.api.java.tuple.Tuple22;
+import org.apache.flink.api.java.tuple.Tuple23;
+import org.apache.flink.api.java.tuple.Tuple24;
+import org.apache.flink.api.java.tuple.Tuple25;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple6;
+import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.tuple.Tuple9;
+
+public class KeySelectorUtil {
+
+ public static Class<?>[] tupleClasses = new Class[] { Tuple1.class, Tuple2.class, Tuple3.class,
+ Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class, Tuple9.class,
+ Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class, Tuple14.class,
+ Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class, Tuple19.class,
+ Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class, Tuple24.class,
+ Tuple25.class };
+
+ public static <X> KeySelector<X, ?> getSelectorForKeys(Keys<X> keys, TypeInformation<X> typeInfo) {
+ int[] logicalKeyPositions = keys.computeLogicalKeyPositions();
+ int keyLength = logicalKeyPositions.length;
+ boolean[] orders = new boolean[keyLength];
+ TypeComparator<X> comparator = ((CompositeType<X>) typeInfo).createComparator(
+ logicalKeyPositions, orders, 0);
+ return new ComparableKeySelector<X>(comparator, keyLength);
+ }
+
+ public static class ComparableKeySelector<IN> implements KeySelector<IN, Tuple> {
+
+ private static final long serialVersionUID = 1L;
+
+ private TypeComparator<IN> comparator;
+ private int keyLength;
+ private Object[] keyArray;
+ private Tuple key;
+
+ public ComparableKeySelector(TypeComparator<IN> comparator, int keyLength) {
+ this.comparator = comparator;
+ this.keyLength = keyLength;
+ keyArray = new Object[keyLength];
+ try {
+ key = (Tuple) tupleClasses[keyLength - 1].newInstance();
+ } catch (Exception e) {
+ }
+ }
+
+ @Override
+ public Tuple getKey(IN value) throws Exception {
+ comparator.extractKeys(value, keyArray, 0);
+ for (int i = 0; i < keyLength; i++) {
+ key.setField(keyArray[i], i);
+ }
+ return key;
+ }
+
+ }
+
+ public static class ArrayKeySelector<IN> implements KeySelector<IN, Tuple> {
+
+ private static final long serialVersionUID = 1L;
+
+ Tuple key;
+ int[] fields;
+
+ public ArrayKeySelector(int... fields) {
+ this.fields = fields;
+ try {
+ key = (Tuple) tupleClasses[fields.length - 1].newInstance();
+ } catch (Exception e) {
+ }
+ }
+
+ @Override
+ public Tuple getKey(IN value) throws Exception {
+ for (int i = 0; i < fields.length; i++) {
+ int pos = fields[i];
+ key.setField(Array.get(value, fields[pos]), i);
+ }
+ return key;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
new file mode 100644
index 0000000..49cd497
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.Keys;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
+import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
+import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
+import org.apache.flink.streaming.api.invokable.operator.GroupedReduceInvokable;
+import org.apache.flink.streaming.api.invokable.operator.StreamReduceInvokable;
+import org.apache.flink.streaming.util.MockContext;
+import org.apache.flink.streaming.util.keys.KeySelectorUtil;
+import org.junit.Test;
+
+public class AggregationFunctionTest {
+
+ @Test
+ public void groupSumIntegerTest() {
+
+ List<Tuple2<Integer, Integer>> expectedSumList = new ArrayList<Tuple2<Integer, Integer>>();
+ List<Tuple2<Integer, Integer>> expectedMinList = new ArrayList<Tuple2<Integer, Integer>>();
+ List<Tuple2<Integer, Integer>> expectedMaxList = new ArrayList<Tuple2<Integer, Integer>>();
+ List<Integer> expectedSumList0 = new ArrayList<Integer>();
+ List<Integer> expectedMinList0 = new ArrayList<Integer>();
+ List<Integer> expectedMaxList0 = new ArrayList<Integer>();
+ List<Tuple2<Integer, Integer>> expectedGroupSumList = new ArrayList<Tuple2<Integer, Integer>>();
+ List<Tuple2<Integer, Integer>> expectedGroupMinList = new ArrayList<Tuple2<Integer, Integer>>();
+ List<Tuple2<Integer, Integer>> expectedGroupMaxList = new ArrayList<Tuple2<Integer, Integer>>();
+
+ List<Integer> simpleInput = new ArrayList<Integer>();
+
+ int groupedSum0 = 0;
+ int groupedSum1 = 0;
+ int groupedSum2 = 0;
+
+ for (int i = 0; i < 9; i++) {
+ simpleInput.add(i);
+ expectedSumList.add(new Tuple2<Integer, Integer>(i % 3, (i + 1) * i / 2));
+ expectedMinList.add(new Tuple2<Integer, Integer>(i % 3, 0));
+ expectedMaxList.add(new Tuple2<Integer, Integer>(i % 3, i));
+
+ expectedSumList0.add((i + 1) * i / 2);
+ expectedMaxList0.add(i);
+ expectedMinList0.add(0);
+
+ int groupedSum;
+ switch (i % 3) {
+ case 0:
+ groupedSum = groupedSum0 += i;
+ break;
+ case 1:
+ groupedSum = groupedSum1 += i;
+ break;
+ default:
+ groupedSum = groupedSum2 += i;
+ break;
+ }
+
+ expectedGroupSumList.add(new Tuple2<Integer, Integer>(i % 3, groupedSum));
+ expectedGroupMinList.add(new Tuple2<Integer, Integer>(i % 3, i % 3));
+ expectedGroupMaxList.add(new Tuple2<Integer, Integer>(i % 3, i));
+ }
+
+ TypeInformation<Tuple2<Integer, Integer>> type1 = TypeExtractor
+ .getForObject(new Tuple2<Integer, Integer>(0, 0));
+ TypeInformation<Integer> type2 = TypeExtractor.getForObject(2);
+
+ ReduceFunction<Tuple2<Integer, Integer>> sumFunction = SumAggregator.getSumFunction(1,
+ Integer.class, type1);
+ ReduceFunction<Integer> sumFunction0 = SumAggregator
+ .getSumFunction(0, Integer.class, type2);
+ ReduceFunction<Tuple2<Integer, Integer>> minFunction = ComparableAggregator.getAggregator(
+ 1, type1, AggregationType.MIN);
+ ReduceFunction<Integer> minFunction0 = ComparableAggregator.getAggregator(0, type2,
+ AggregationType.MIN);
+ ReduceFunction<Tuple2<Integer, Integer>> maxFunction = ComparableAggregator.getAggregator(
+ 1, type1, AggregationType.MAX);
+ ReduceFunction<Integer> maxFunction0 = ComparableAggregator.getAggregator(0, type2,
+ AggregationType.MAX);
+ List<Tuple2<Integer, Integer>> sumList = MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(sumFunction), getInputList());
+
+ List<Tuple2<Integer, Integer>> minList = MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(minFunction), getInputList());
+
+ List<Tuple2<Integer, Integer>> maxList = MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxFunction), getInputList());
+
+ TypeInformation<Tuple2<Integer, Integer>> typeInfo = TypeExtractor
+ .getForObject(new Tuple2<Integer, Integer>(1, 1));
+
+ KeySelector<Tuple2<Integer, Integer>, ?> keySelector = KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys<Tuple2<Integer, Integer>>(new int[] { 0 }, typeInfo),
+ typeInfo);
+
+ List<Tuple2<Integer, Integer>> groupedSumList = MockContext.createAndExecute(
+ new GroupedReduceInvokable<Tuple2<Integer, Integer>>(sumFunction, keySelector),
+ getInputList());
+
+ List<Tuple2<Integer, Integer>> groupedMinList = MockContext.createAndExecute(
+ new GroupedReduceInvokable<Tuple2<Integer, Integer>>(minFunction, keySelector),
+ getInputList());
+
+ List<Tuple2<Integer, Integer>> groupedMaxList = MockContext.createAndExecute(
+ new GroupedReduceInvokable<Tuple2<Integer, Integer>>(maxFunction, keySelector),
+ getInputList());
+
+ assertEquals(expectedSumList, sumList);
+ assertEquals(expectedMinList, minList);
+ assertEquals(expectedMaxList, maxList);
+ assertEquals(expectedGroupSumList, groupedSumList);
+ assertEquals(expectedGroupMinList, groupedMinList);
+ assertEquals(expectedGroupMaxList, groupedMaxList);
+ assertEquals(expectedSumList0, MockContext.createAndExecute(
+ new StreamReduceInvokable<Integer>(sumFunction0), simpleInput));
+ assertEquals(expectedMinList0, MockContext.createAndExecute(
+ new StreamReduceInvokable<Integer>(minFunction0), simpleInput));
+ assertEquals(expectedMaxList0, MockContext.createAndExecute(
+ new StreamReduceInvokable<Integer>(maxFunction0), simpleInput));
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
+ try {
+ env.generateSequence(1, 100).min(1);
+ fail();
+ } catch (Exception e) {
+ // Nothing to do here
+ }
+ try {
+ env.generateSequence(1, 100).min(2);
+ fail();
+ } catch (Exception e) {
+ // Nothing to do here
+ }
+ try {
+ env.generateSequence(1, 100).min(3);
+ fail();
+ } catch (Exception e) {
+ // Nothing to do here
+ }
+
+ ReduceFunction<Tuple2<Integer, Integer>> maxByFunctionFirst = ComparableAggregator
+ .getAggregator(0, type1, AggregationType.MAXBY, true);
+ ReduceFunction<Tuple2<Integer, Integer>> maxByFunctionLast = ComparableAggregator
+ .getAggregator(0, type1, AggregationType.MAXBY, false);
+
+ ReduceFunction<Tuple2<Integer, Integer>> minByFunctionFirst = ComparableAggregator
+ .getAggregator(0, type1, AggregationType.MINBY, true);
+ ReduceFunction<Tuple2<Integer, Integer>> minByFunctionLast = ComparableAggregator
+ .getAggregator(0, type1, AggregationType.MINBY, false);
+
+ List<Tuple2<Integer, Integer>> maxByFirstExpected = new ArrayList<Tuple2<Integer, Integer>>();
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(1, 1));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+
+ List<Tuple2<Integer, Integer>> maxByLastExpected = new ArrayList<Tuple2<Integer, Integer>>();
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(1, 1));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 8));
+
+ List<Tuple2<Integer, Integer>> minByFirstExpected = new ArrayList<Tuple2<Integer, Integer>>();
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+
+ List<Tuple2<Integer, Integer>> minByLastExpected = new ArrayList<Tuple2<Integer, Integer>>();
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
+
+ assertEquals(maxByFirstExpected, MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxByFunctionFirst),
+ getInputList()));
+ assertEquals(maxByLastExpected, MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxByFunctionLast),
+ getInputList()));
+ assertEquals(minByLastExpected, MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(minByFunctionLast),
+ getInputList()));
+ assertEquals(minByFirstExpected, MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(minByFunctionFirst),
+ getInputList()));
+
+ }
+
+ @Test
+ public void minMaxByTest() {
+ TypeInformation<Tuple2<Integer, Integer>> type1 = TypeExtractor
+ .getForObject(new Tuple2<Integer, Integer>(0, 0));
+
+ ReduceFunction<Tuple2<Integer, Integer>> maxByFunctionFirst = ComparableAggregator
+ .getAggregator(0, type1, AggregationType.MAXBY, true);
+ ReduceFunction<Tuple2<Integer, Integer>> maxByFunctionLast = ComparableAggregator
+ .getAggregator(0, type1, AggregationType.MAXBY, false);
+
+ ReduceFunction<Tuple2<Integer, Integer>> minByFunctionFirst = ComparableAggregator
+ .getAggregator(0, type1, AggregationType.MINBY, true);
+ ReduceFunction<Tuple2<Integer, Integer>> minByFunctionLast = ComparableAggregator
+ .getAggregator(0, type1, AggregationType.MINBY, false);
+
+ List<Tuple2<Integer, Integer>> maxByFirstExpected = new ArrayList<Tuple2<Integer, Integer>>();
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(1, 1));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
+
+ List<Tuple2<Integer, Integer>> maxByLastExpected = new ArrayList<Tuple2<Integer, Integer>>();
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(1, 1));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
+ maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 8));
+
+ List<Tuple2<Integer, Integer>> minByFirstExpected = new ArrayList<Tuple2<Integer, Integer>>();
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
+
+ List<Tuple2<Integer, Integer>> minByLastExpected = new ArrayList<Tuple2<Integer, Integer>>();
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
+ minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
+
+ assertEquals(maxByFirstExpected, MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxByFunctionFirst),
+ getInputList()));
+ assertEquals(maxByLastExpected, MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxByFunctionLast),
+ getInputList()));
+ assertEquals(minByLastExpected, MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(minByFunctionLast),
+ getInputList()));
+ assertEquals(minByFirstExpected, MockContext.createAndExecute(
+ new StreamReduceInvokable<Tuple2<Integer, Integer>>(minByFunctionFirst),
+ getInputList()));
+ }
+
+ private List<Tuple2<Integer, Integer>> getInputList() {
+ ArrayList<Tuple2<Integer, Integer>> inputList = new ArrayList<Tuple2<Integer, Integer>>();
+ for (int i = 0; i < 9; i++) {
+ inputList.add(new Tuple2<Integer, Integer>(i % 3, i));
+ }
+ return inputList;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java
new file mode 100644
index 0000000..6ad827a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.IterativeDataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class IterateTest {
+
+ private static final long MEMORYSIZE = 32;
+ private static boolean iterated = false;
+
+ public static final class IterationHead extends RichFlatMapFunction<Boolean, Boolean> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(Boolean value, Collector<Boolean> out) throws Exception {
+ if (value) {
+ iterated = true;
+ } else {
+ out.collect(value);
+ }
+
+ }
+
+ }
+
+ public static final class IterationTail extends RichFlatMapFunction<Boolean, Boolean> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(Boolean value, Collector<Boolean> out) throws Exception {
+ out.collect(true);
+
+ }
+
+ }
+
+ public static final class MySink implements SinkFunction<Boolean> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void invoke(Boolean tuple) {
+ }
+
+ }
+
+ @Test
+ public void test() throws Exception {
+ StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
+
+ env.setBufferTimeout(10);
+
+ DataStream<Boolean> source = env.fromElements(false, false, false);
+
+ IterativeDataStream<Boolean> iteration = source.iterate(3000);
+
+ DataStream<Boolean> increment = iteration.flatMap(new IterationHead()).flatMap(
+ new IterationTail());
+
+ iteration.closeWith(increment).addSink(new MySink());
+
+ env.execute();
+
+ assertTrue(iterated);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java
new file mode 100644
index 0000000..2486715
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class OutputSplitterTest {
+
+ private static final long MEMORYSIZE = 32;
+
+ private static ArrayList<Integer> splitterResult1 = new ArrayList<Integer>();
+ private static ArrayList<Integer> splitterResult2 = new ArrayList<Integer>();
+
+
+ private static ArrayList<Integer> expectedSplitterResult = new ArrayList<Integer>();
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testOnMergedDataStream() throws Exception {
+ splitterResult1.clear();
+ splitterResult2.clear();
+
+ StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
+ env.setBufferTimeout(1);
+
+ DataStream<Integer> d1 = env.fromElements(0,2,4,6,8);
+ DataStream<Integer> d2 = env.fromElements(1,3,5,7,9);
+
+ d1 = d1.merge(d2);
+
+ d1.split(new OutputSelector<Integer>() {
+ private static final long serialVersionUID = 8354166915727490130L;
+
+ @Override
+ public Iterable<String> select(Integer value) {
+ List<String> s = new ArrayList<String>();
+ if (value > 4) {
+ s.add(">");
+ } else {
+ s.add("<");
+ }
+ return s;
+ }
+ }).select(">").addSink(new SinkFunction<Integer>() {
+
+ private static final long serialVersionUID = 5827187510526388104L;
+
+ @Override
+ public void invoke(Integer value) {
+ splitterResult1.add(value);
+ }
+ });
+
+ d1.split(new OutputSelector<Integer>() {
+ private static final long serialVersionUID = -6822487543355994807L;
+
+ @Override
+ public Iterable<String> select(Integer value) {
+ List<String> s = new ArrayList<String>();
+ if (value % 3 == 0) {
+ s.add("yes");
+ } else {
+ s.add("no");
+ }
+ return s;
+ }
+ }).select("yes").addSink(new SinkFunction<Integer>() {
+ private static final long serialVersionUID = -2674335071267854599L;
+
+ @Override
+ public void invoke(Integer value) {
+ splitterResult2.add(value);
+ }
+ });
+ env.execute();
+
+ Collections.sort(splitterResult1);
+ Collections.sort(splitterResult2);
+
+ expectedSplitterResult.clear();
+ expectedSplitterResult.addAll(Arrays.asList(5,6,7,8,9));
+ assertEquals(expectedSplitterResult, splitterResult1);
+
+ expectedSplitterResult.clear();
+ expectedSplitterResult.addAll(Arrays.asList(0,3,6,9));
+ assertEquals(expectedSplitterResult, splitterResult2);
+ }
+
+ @Test
+ public void testOnSingleDataStream() throws Exception {
+ splitterResult1.clear();
+ splitterResult2.clear();
+
+ StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
+ env.setBufferTimeout(1);
+
+ DataStream<Integer> ds = env.fromElements(0,1,2,3,4,5,6,7,8,9);
+
+ ds.split(new OutputSelector<Integer>() {
+ private static final long serialVersionUID = 2524335410904414121L;
+
+ @Override
+ public Iterable<String> select(Integer value) {
+ List<String> s = new ArrayList<String>();
+ if (value % 2 == 0) {
+ s.add("even");
+ } else {
+ s.add("odd");
+ }
+ return s;
+ }
+ }).select("even").addSink(new SinkFunction<Integer>() {
+
+ private static final long serialVersionUID = -2995092337537209535L;
+
+ @Override
+ public void invoke(Integer value) {
+ splitterResult1.add(value);
+ }
+ });
+
+ ds.split(new OutputSelector<Integer>() {
+
+ private static final long serialVersionUID = -511693919586034092L;
+
+ @Override
+ public Iterable<String> select(Integer value) {
+ List<String> s = new ArrayList<String>();
+ if (value % 4 == 0) {
+ s.add("yes");
+ } else {
+ s.add("no");
+ }
+ return s;
+ }
+ }).select("yes").addSink(new SinkFunction<Integer>() {
+
+ private static final long serialVersionUID = -1749077049727705424L;
+
+ @Override
+ public void invoke(Integer value) {
+ splitterResult2.add(value);
+ }
+ });
+ env.execute();
+
+ Collections.sort(splitterResult1);
+ Collections.sort(splitterResult2);
+
+ expectedSplitterResult.clear();
+ expectedSplitterResult.addAll(Arrays.asList(0,2,4,6,8));
+ assertEquals(expectedSplitterResult, splitterResult1);
+
+ expectedSplitterResult.clear();
+ expectedSplitterResult.addAll(Arrays.asList(0,4,8));
+ assertEquals(expectedSplitterResult, splitterResult2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
new file mode 100755
index 0000000..757f6f6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
+import org.junit.Test;
+
+public class PrintTest implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private static final long MEMORYSIZE = 32;
+
+ private static final class IdentityMap implements MapFunction<Long, Long> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Long map(Long value) throws Exception {
+ return value;
+ }
+ }
+
+ private static final class FilterAll implements FilterFunction<Long> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean filter(Long value) throws Exception {
+ return true;
+ }
+ }
+
+ @Test
+ public void test() throws Exception {
+ StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
+ env.generateSequence(1, 10).map(new IdentityMap()).filter(new FilterAll()).print();
+ env.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java
new file mode 100644
index 0000000..9be7de6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayInputStream;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.function.source.FromElementsFunction;
+import org.apache.flink.streaming.api.function.source.SocketTextStreamFunction;
+import org.apache.flink.streaming.util.MockCollector;
+import org.apache.flink.streaming.util.MockSource;
+import org.junit.Test;
+
+public class SourceTest {
+
+ @Test
+ public void fromElementsTest() {
+ List<Integer> expectedList = Arrays.asList(1, 2, 3);
+ List<Integer> actualList = MockSource.createAndExecute(new FromElementsFunction<Integer>(1,
+ 2, 3));
+ assertEquals(expectedList, actualList);
+ }
+
+ @Test
+ public void fromCollectionTest() {
+ List<Integer> expectedList = Arrays.asList(1, 2, 3);
+ List<Integer> actualList = MockSource.createAndExecute(new FromElementsFunction<Integer>(
+ Arrays.asList(1, 2, 3)));
+ assertEquals(expectedList, actualList);
+ }
+
+ @Test
+ public void socketTextStreamTest() throws Exception {
+ List<String> expectedList = Arrays.asList("a", "b", "c");
+ List<String> actualList = new ArrayList<String>();
+
+ byte[] data = { 'a', '\n', 'b', '\n', 'c', '\n' };
+
+ Socket socket = mock(Socket.class);
+ when(socket.getInputStream()).thenReturn(new ByteArrayInputStream(data));
+ when(socket.isClosed()).thenReturn(false);
+ when(socket.isConnected()).thenReturn(true);
+
+ new SocketTextStreamFunction("", 0, '\n').streamFromSocket(new MockCollector<String>(
+ actualList), socket);
+ assertEquals(expectedList, actualList);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
new file mode 100644
index 0000000..3da6b5f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
+import org.junit.Test;
+
+public class WindowCrossJoinTest implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final long MEMORYSIZE = 32;
+
+ private static ArrayList<Tuple2<Tuple2<Integer, String>, Integer>> joinResults = new ArrayList<Tuple2<Tuple2<Integer, String>, Integer>>();
+ private static ArrayList<Tuple2<Tuple2<Integer, String>, Integer>> joinExpectedResults = new ArrayList<Tuple2<Tuple2<Integer, String>, Integer>>();
+
+ private static ArrayList<Tuple2<Tuple2<Integer, String>, Integer>> crossResults = new ArrayList<Tuple2<Tuple2<Integer, String>, Integer>>();
+ private static ArrayList<Tuple2<Tuple2<Integer, String>, Integer>> crossExpectedResults = new ArrayList<Tuple2<Tuple2<Integer, String>, Integer>>();
+
+ @Test
+ public void test() throws Exception {
+ StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
+ env.setBufferTimeout(1);
+
+ ArrayList<Tuple2<Integer, String>> in1 = new ArrayList<Tuple2<Integer, String>>();
+ ArrayList<Tuple1<Integer>> in2 = new ArrayList<Tuple1<Integer>>();
+
+ in1.add(new Tuple2<Integer, String>(10, "a"));
+ in1.add(new Tuple2<Integer, String>(20, "b"));
+ in1.add(new Tuple2<Integer, String>(20, "x"));
+ in1.add(new Tuple2<Integer, String>(0, "y"));
+
+ in2.add(new Tuple1<Integer>(0));
+ in2.add(new Tuple1<Integer>(5));
+ in2.add(new Tuple1<Integer>(20));
+
+ joinExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(20, "b"), 20));
+ joinExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(20, "x"), 20));
+ joinExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(0, "y"), 0));
+
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(10, "a"), 0));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(10, "a"), 5));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(10, "a"), 20));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(20, "b"), 0));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(20, "b"), 5));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(20, "b"), 20));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(20, "x"), 0));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(20, "x"), 5));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(20, "x"), 20));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(0, "y"), 0));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(0, "y"), 5));
+ crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
+ new Tuple2<Integer, String>(0, "y"), 20));
+
+ DataStream<Tuple2<Integer, String>> inStream1 = env.fromCollection(in1);
+ DataStream<Tuple1<Integer>> inStream2 = env.fromCollection(in2);
+
+ inStream1
+ .join(inStream2)
+ .onWindow(1000, new MyTimestamp<Tuple2<Integer, String>>(),
+ new MyTimestamp<Tuple1<Integer>>(), 100).where(0).equalTo(0)
+ .addSink(new JoinResultSink());
+
+ inStream1
+ .cross(inStream2)
+ .onWindow(1000, new MyTimestamp<Tuple2<Integer, String>>(),
+ new MyTimestamp<Tuple1<Integer>>(), 100)
+ .with(new CrossFunction<Tuple2<Integer, String>, Tuple1<Integer>, Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<Tuple2<Integer, String>, Tuple1<Integer>> cross(
+ Tuple2<Integer, String> val1, Tuple1<Integer> val2) throws Exception {
+ return new Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>(val1, val2);
+ }
+ }).addSink(new CrossResultSink());
+
+ env.execute();
+
+ assertEquals(joinExpectedResults, joinResults);
+ assertEquals(crossExpectedResults, crossResults);
+ }
+
+ private static class MyTimestamp<T> implements Timestamp<T> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(T value) {
+ return 101L;
+ }
+ }
+
+ private static class JoinResultSink implements
+ SinkFunction<Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void invoke(Tuple2<Tuple2<Integer, String>, Tuple1<Integer>> value) {
+ joinResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(value.f0, value.f1.f0));
+ }
+ }
+
+ private static class CrossResultSink implements
+ SinkFunction<Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void invoke(Tuple2<Tuple2<Integer, String>, Tuple1<Integer>> value) {
+ crossResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(value.f0, value.f1.f0));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java
new file mode 100644
index 0000000..78cbbe5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import org.apache.flink.streaming.api.datastream.SplitDataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class DirectedOutputTest {
+
+ private static final String TEN = "ten";
+ private static final String ODD = "odd";
+ private static final String ALL = "all";
+ private static final String EVEN_AND_ODD = "evenAndOdd";
+ private static final String ODD_AND_TEN = "oddAndTen";
+ private static final String EVEN = "even";
+ private static final String NON_SELECTED = "nonSelected";
+
+ static final class MyOutputSelector implements OutputSelector<Long> {
+ private static final long serialVersionUID = 1L;
+
+ List<String> outputs = new ArrayList<String>();
+
+ @Override
+ public Iterable<String> select(Long value) {
+ outputs.clear();
+ if (value % 2 == 0) {
+ outputs.add(EVEN);
+ } else {
+ outputs.add(ODD);
+ }
+
+ if (value == 10L) {
+ outputs.add(TEN);
+ }
+
+ if (value == 11L) {
+ outputs.add(NON_SELECTED);
+ }
+ return outputs;
+ }
+ }
+
+ static final class ListSink implements SinkFunction<Long> {
+ private static final long serialVersionUID = 1L;
+
+ private String name;
+ private transient List<Long> list;
+
+ public ListSink(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public void invoke(Long value) {
+ list.add(value);
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws IOException,
+ ClassNotFoundException {
+ in.defaultReadObject();
+ outputs.put(name, new ArrayList<Long>());
+ this.list = outputs.get(name);
+ }
+ }
+
+ private static Map<String, List<Long>> outputs = new HashMap<String, List<Long>>();
+
+ @Test
+ public void outputSelectorTest() throws Exception {
+ StreamExecutionEnvironment env = new TestStreamEnvironment(1, 128);
+
+ SplitDataStream<Long> source = env.generateSequence(1, 11).split(new MyOutputSelector());
+ source.select(EVEN).addSink(new ListSink(EVEN));
+ source.select(ODD, TEN).addSink(new ListSink(ODD_AND_TEN));
+ source.select(EVEN, ODD).addSink(new ListSink(EVEN_AND_ODD));
+ source.addSink(new ListSink(ALL));
+
+ env.execute();
+ assertEquals(Arrays.asList(2L, 4L, 6L, 8L, 10L), outputs.get(EVEN));
+ assertEquals(Arrays.asList(1L, 3L, 5L, 7L, 9L, 10L, 11L), outputs.get(ODD_AND_TEN));
+ assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
+ outputs.get(EVEN_AND_ODD));
+ assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L), outputs.get(ALL));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java
new file mode 100755
index 0000000..1615a45
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.junit.Test;
+
+public class OutputSelectorTest {
+
+ static final class MyOutputSelector implements OutputSelector<Tuple1<Integer>> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Iterable<String> select(Tuple1<Integer> tuple) {
+
+ String[] outputs = new String[tuple.f0];
+
+ for (Integer i = 0; i < tuple.f0; i++) {
+ outputs[i] = i.toString();
+ }
+ return Arrays.asList(outputs);
+ }
+ }
+
+ @Test
+ public void testGetOutputs() {
+ OutputSelector<Tuple1<Integer>> selector = new MyOutputSelector();
+ List<String> expectedOutputs = new ArrayList<String>();
+ expectedOutputs.add("0");
+ expectedOutputs.add("1");
+ assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(2)));
+ expectedOutputs.add("2");
+ assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(3)));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
new file mode 100755
index 0000000..49b3bf8
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.streamvertex.MockRecordWriter;
+import org.apache.flink.streaming.util.MockRecordWriterFactory;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class StreamCollectorTest {
+
+ @Test
+ public void testCollect() {
+ MockRecordWriter recWriter = MockRecordWriterFactory.create();
+ SerializationDelegate<StreamRecord<Tuple1<Integer>>> sd = new SerializationDelegate<StreamRecord<Tuple1<Integer>>>(
+ null);
+ sd.setInstance(new StreamRecord<Tuple1<Integer>>().setObject(new Tuple1<Integer>()));
+
+ Collector<Tuple1<Integer>> collector = new StreamOutput<Tuple1<Integer>>(recWriter, 2, sd);
+ collector.collect(new Tuple1<Integer>(3));
+ collector.collect(new Tuple1<Integer>(4));
+ collector.collect(new Tuple1<Integer>(5));
+ collector.collect(new Tuple1<Integer>(6));
+
+ assertArrayEquals(new Integer[] { 3, 4, 5, 6 }, recWriter.emittedRecords.toArray());
+ }
+
+ @Test
+ public void testClose() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java
new file mode 100644
index 0000000..a91bd0c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoFlatMapInvokable;
+import org.apache.flink.streaming.util.MockCoContext;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class CoFlatMapTest implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final static class MyCoFlatMap implements CoFlatMapFunction<String, Integer, String> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap1(String value, Collector<String> coll) {
+ for (int i = 0; i < value.length(); i++) {
+ coll.collect(value.substring(i, i + 1));
+ }
+ }
+
+ @Override
+ public void flatMap2(Integer value, Collector<String> coll) {
+ coll.collect(value.toString());
+ }
+ }
+
+ @Test
+ public void coFlatMapTest() {
+ CoFlatMapInvokable<String, Integer, String> invokable = new CoFlatMapInvokable<String, Integer, String>(
+ new MyCoFlatMap());
+
+ List<String> expectedList = Arrays.asList("a", "b", "c", "1", "d", "e", "f", "2", "g", "h",
+ "e", "3", "4", "5");
+ List<String> actualList = MockCoContext.createAndExecute(invokable,
+ Arrays.asList("abc", "def", "ghe"), Arrays.asList(1, 2, 3, 4, 5));
+
+ assertEquals(expectedList, actualList);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void multipleInputTest() {
+ LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+ DataStream<Integer> ds1 = env.fromElements(1, 3, 5);
+ DataStream<Integer> ds2 = env.fromElements(2, 4).merge(ds1);
+
+ try {
+ ds1.forward().merge(ds2);
+ fail();
+ } catch (RuntimeException e) {
+ // expected
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
new file mode 100644
index 0000000..273bbae
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.streaming.api.function.co.CoReduceFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedReduceInvokable;
+import org.apache.flink.streaming.util.MockCoContext;
+import org.junit.Test;
+
+public class CoGroupedReduceTest {
+
+ private final static class MyCoReduceFunction implements
+ CoReduceFunction<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple3<String, String, String> reduce1(Tuple3<String, String, String> value1,
+ Tuple3<String, String, String> value2) {
+ return new Tuple3<String, String, String>(value1.f0, value1.f1 + value2.f1, value1.f2);
+ }
+
+ @Override
+ public Tuple2<Integer, Integer> reduce2(Tuple2<Integer, Integer> value1,
+ Tuple2<Integer, Integer> value2) {
+ return new Tuple2<Integer, Integer>(value1.f0, value1.f1 + value2.f1);
+ }
+
+ @Override
+ public String map1(Tuple3<String, String, String> value) {
+ return value.f1;
+ }
+
+ @Override
+ public String map2(Tuple2<Integer, Integer> value) {
+ return value.f1.toString();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void coGroupedReduceTest() {
+ Tuple3<String, String, String> word1 = new Tuple3<String, String, String>("a", "word1", "b");
+ Tuple3<String, String, String> word2 = new Tuple3<String, String, String>("b", "word2", "a");
+ Tuple3<String, String, String> word3 = new Tuple3<String, String, String>("a", "word3", "a");
+ Tuple2<Integer, Integer> int1 = new Tuple2<Integer, Integer>(2, 1);
+ Tuple2<Integer, Integer> int2 = new Tuple2<Integer, Integer>(1, 2);
+ Tuple2<Integer, Integer> int3 = new Tuple2<Integer, Integer>(0, 3);
+ Tuple2<Integer, Integer> int4 = new Tuple2<Integer, Integer>(2, 4);
+ Tuple2<Integer, Integer> int5 = new Tuple2<Integer, Integer>(1, 5);
+
+ KeySelector<Tuple3<String, String, String>, ?> keySelector0 = new KeySelector<Tuple3<String, String, String>, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple3<String, String, String> value) throws Exception {
+ return value.f0;
+ }
+ };
+
+ KeySelector<Tuple2<Integer, Integer>, ?> keySelector1 = new KeySelector<Tuple2<Integer, Integer>, Integer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Tuple2<Integer, Integer> value) throws Exception {
+ return value.f0;
+ }
+ };
+
+ KeySelector<Tuple3<String, String, String>, ?> keySelector2 = new KeySelector<Tuple3<String, String, String>, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple3<String, String, String> value) throws Exception {
+ return value.f2;
+ }
+ };
+
+ CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> invokable = new CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
+ new MyCoReduceFunction(), keySelector0, keySelector1);
+
+ List<String> expected = Arrays.asList("word1", "1", "word2", "2", "word1word3", "3", "5",
+ "7");
+
+ List<String> actualList = MockCoContext.createAndExecute(invokable,
+ Arrays.asList(word1, word2, word3), Arrays.asList(int1, int2, int3, int4, int5));
+
+ assertEquals(expected, actualList);
+
+ invokable = new CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
+ new MyCoReduceFunction(), keySelector2, keySelector1);
+
+ expected = Arrays.asList("word1", "1", "word2", "2", "word2word3", "3", "5", "7");
+
+ actualList = MockCoContext.createAndExecute(invokable, Arrays.asList(word1, word2, word3),
+ Arrays.asList(int1, int2, int3, int4, int5));
+
+ assertEquals(expected, actualList);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java
new file mode 100644
index 0000000..93d1741
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.function.co.CoMapFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoMapInvokable;
+import org.apache.flink.streaming.util.MockCoContext;
+import org.junit.Test;
+
+public class CoMapTest implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final static class MyCoMap implements CoMapFunction<Double, Integer, String> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String map1(Double value) {
+ return value.toString();
+ }
+
+ @Override
+ public String map2(Integer value) {
+ return value.toString();
+ }
+ }
+
+ @Test
+ public void coMapTest() {
+ CoMapInvokable<Double, Integer, String> invokable = new CoMapInvokable<Double, Integer, String>(new MyCoMap());
+
+ List<String> expectedList = Arrays.asList("1.1", "1", "1.2", "2", "1.3", "3", "1.4", "1.5");
+ List<String> actualList = MockCoContext.createAndExecute(invokable, Arrays.asList(1.1, 1.2, 1.3, 1.4, 1.5), Arrays.asList(1, 2, 3));
+
+ assertEquals(expectedList, actualList);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java
new file mode 100755
index 0000000..3343ba0
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.function.co.CoReduceFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoReduceInvokable;
+import org.apache.flink.streaming.util.MockCoContext;
+import org.junit.Test;
+
+public class CoStreamReduceTest {
+
+ public static class MyCoReduceFunction implements
+ CoReduceFunction<Integer, String, Integer> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce1(Integer value1, Integer value2) {
+ return value1 * value2;
+ }
+
+ @Override
+ public String reduce2(String value1, String value2) {
+ return value1 + value2;
+ }
+
+ @Override
+ public Integer map1(Integer value) {
+ return value;
+ }
+
+ @Override
+ public Integer map2(String value) {
+ return Integer.parseInt(value);
+ }
+
+ }
+
+ @Test
+ public void coStreamReduceTest() {
+
+ CoReduceInvokable<Integer, String, Integer> coReduce = new CoReduceInvokable<Integer, String, Integer>(
+ new MyCoReduceFunction());
+
+ List<Integer> expected1 = Arrays.asList(1, 9, 2, 99, 6, 998, 24);
+ List<Integer> result = MockCoContext.createAndExecute(coReduce,
+ Arrays.asList(1, 2, 3, 4), Arrays.asList("9", "9", "8"));
+
+ assertEquals(expected1, result);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
new file mode 100644
index 0000000..4ab3492
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.function.co.CoWindowFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.apache.flink.streaming.util.MockCoContext;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class CoWindowTest {
+
+ public static final class MyCoGroup1 implements CoWindowFunction<Integer, Integer, Integer> {
+
+ private static final long serialVersionUID = 1L;
+
+ @SuppressWarnings("unused")
+ @Override
+ public void coWindow(List<Integer> first, List<Integer> second, Collector<Integer> out)
+ throws Exception {
+ Integer count1 = 0;
+ for (Integer i : first) {
+ count1++;
+ }
+ Integer count2 = 0;
+ for (Integer i : second) {
+ count2++;
+ }
+ out.collect(count1);
+ out.collect(count2);
+
+ }
+
+ }
+
+ public static final class MyCoGroup2 implements
+ CoWindowFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void coWindow(List<Tuple2<Integer, Integer>> first,
+ List<Tuple2<Integer, Integer>> second, Collector<Integer> out) throws Exception {
+
+ Set<Integer> firstElements = new HashSet<Integer>();
+ for (Tuple2<Integer, Integer> value : first) {
+ firstElements.add(value.f1);
+ }
+ for (Tuple2<Integer, Integer> value : second) {
+ if (firstElements.contains(value.f1)) {
+ out.collect(value.f1);
+ }
+ }
+
+ }
+
+ }
+
+ private static final class MyTS1 implements Timestamp<Integer> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(Integer value) {
+ return value;
+ }
+
+ }
+
+ private static final class MyTS2 implements Timestamp<Tuple2<Integer, Integer>> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(Tuple2<Integer, Integer> value) {
+ return value.f0;
+ }
+
+ }
+
+ @Test
+ public void coWindowGroupReduceTest2() throws Exception {
+
+ CoWindowInvokable<Integer, Integer, Integer> invokable1 = new CoWindowInvokable<Integer, Integer, Integer>(
+ new MyCoGroup1(), 2, 1, new TimestampWrapper<Integer>(new MyTS1(), 1),
+ new TimestampWrapper<Integer>(new MyTS1(), 1));
+
+ // Windowsize 2, slide 1
+ // 1,2|2,3|3,4|4,5
+
+ List<Integer> input11 = new ArrayList<Integer>();
+ input11.add(1);
+ input11.add(1);
+ input11.add(2);
+ input11.add(3);
+ input11.add(3);
+
+ List<Integer> input12 = new ArrayList<Integer>();
+ input12.add(1);
+ input12.add(2);
+ input12.add(3);
+ input12.add(3);
+ input12.add(5);
+
+ // Windows: (1,1,2)(1,1,2)|(2,3,3)(2,3,3)|(3,3)(3,3)|(5)(5)
+ // expected output: 3,2|3,3|2,2|0,1
+
+ List<Integer> expected1 = new ArrayList<Integer>();
+ expected1.add(3);
+ expected1.add(2);
+ expected1.add(3);
+ expected1.add(3);
+ expected1.add(2);
+ expected1.add(2);
+ expected1.add(0);
+ expected1.add(1);
+
+ List<Integer> actual1 = MockCoContext.createAndExecute(invokable1, input11, input12);
+ assertEquals(expected1, actual1);
+
+ CoWindowInvokable<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> invokable2 = new CoWindowInvokable<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer>(
+ new MyCoGroup2(), 2, 3, new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(),
+ 1), new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(), 1));
+
+ // WindowSize 2, slide 3
+ // 1,2|4,5|7,8|
+
+ List<Tuple2<Integer, Integer>> input21 = new ArrayList<Tuple2<Integer, Integer>>();
+ input21.add(new Tuple2<Integer, Integer>(1, 1));
+ input21.add(new Tuple2<Integer, Integer>(1, 2));
+ input21.add(new Tuple2<Integer, Integer>(2, 3));
+ input21.add(new Tuple2<Integer, Integer>(3, 4));
+ input21.add(new Tuple2<Integer, Integer>(3, 5));
+ input21.add(new Tuple2<Integer, Integer>(4, 6));
+ input21.add(new Tuple2<Integer, Integer>(4, 7));
+ input21.add(new Tuple2<Integer, Integer>(5, 8));
+
+ List<Tuple2<Integer, Integer>> input22 = new ArrayList<Tuple2<Integer, Integer>>();
+ input22.add(new Tuple2<Integer, Integer>(1, 1));
+ input22.add(new Tuple2<Integer, Integer>(2, 0));
+ input22.add(new Tuple2<Integer, Integer>(2, 2));
+ input22.add(new Tuple2<Integer, Integer>(3, 9));
+ input22.add(new Tuple2<Integer, Integer>(3, 4));
+ input22.add(new Tuple2<Integer, Integer>(4, 10));
+ input22.add(new Tuple2<Integer, Integer>(5, 8));
+ input22.add(new Tuple2<Integer, Integer>(5, 7));
+
+ List<Integer> expected2 = new ArrayList<Integer>();
+ expected2.add(1);
+ expected2.add(2);
+ expected2.add(8);
+ expected2.add(7);
+
+ List<Integer> actual2 = MockCoContext.createAndExecute(invokable2, input21, input22);
+ assertEquals(expected2, actual2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokableTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokableTest.java
new file mode 100644
index 0000000..969a06b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokableTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class CounterInvokableTest {
+
+ @Test
+ public void counterTest() {
+ CounterInvokable<String> invokable = new CounterInvokable<String>();
+
+ List<Long> expected = Arrays.asList(1L, 2L, 3L);
+ List<Long> actual = MockContext.createAndExecute(invokable, Arrays.asList("one", "two", "three"));
+
+ assertEquals(expected, actual);
+ }
+}
[43/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
deleted file mode 100644
index 3cfd7d4..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.flume;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.util.SerializationSchema;
-import org.apache.flink.streaming.connectors.util.SimpleStringSchema;
-
-public class FlumeTopology {
-
- public static void main(String[] args) throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- @SuppressWarnings("unused")
- DataStream<String> dataStream1 = env.addSource(
- new FlumeSource<String>("localhost", 41414, new SimpleStringSchema())).addSink(
- new FlumeSink<String>("localhost", 42424, new StringToByteSerializer()));
-
- env.execute();
- }
-
- public static class StringToByteSerializer implements SerializationSchema<String, byte[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public byte[] serialize(String element) {
- return element.getBytes();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
deleted file mode 100644
index 0f16541..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.json;
-
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.sling.commons.json.JSONException;
-
-/**
- * Abstract class derived from {@link RichFlatMapFunction} to handle JSON files.
- *
- * @param <IN>
- * Type of the input elements.
- * @param <OUT>
- * Type of the returned elements.
- */
-public abstract class JSONParseFlatMap<IN, OUT> extends RichFlatMapFunction<IN, OUT> {
-
- private static final long serialVersionUID = 1L;
-
- // private static final Log LOG = LogFactory.getLog(JSONParseFlatMap.class);
-
- /**
- * Get the value object associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public Object get(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).get("retValue");
- }
-
- /**
- * Get the boolean value associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public boolean getBoolean(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).getBoolean("retValue");
- }
-
- /**
- * Get the double value associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public double getDouble(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).getDouble("retValue");
- }
-
- /**
- * Get the int value associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public int getInt(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).getInt("retValue");
- }
-
- /**
- * Get the long value associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public long getLong(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).getLong("retValue");
- }
-
- /**
- * Get the String value associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public String getString(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).getString("retValue");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
deleted file mode 100644
index c1eabbd..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.json;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.StringTokenizer;
-
-import org.apache.sling.commons.json.JSONArray;
-import org.apache.sling.commons.json.JSONException;
-import org.apache.sling.commons.json.JSONObject;
-
-/**
- * A JSONParser contains a JSONObject and provides opportunity to access
- * embedded fields in JSON code.
- */
-public class JSONParser {
-
- private JSONObject originalJO;
- private String searchedfield;
- private Object temp;
-
- /**
- * Construct a JSONParser from a string. The string has to be a JSON code
- * from which we want to get a field.
- *
- * @param jsonText
- * A string which contains a JSON code. String representation of
- * a JSON code.
- * @throws JSONException
- * If there is a syntax error in the source string.
- */
- public JSONParser(String jsonText) throws JSONException {
- originalJO = new JSONObject(jsonText);
- }
-
- /**
- *
- * Parse the JSON code passed to the constructor to find the given key.
- *
- * @param key
- * The key whose value is searched for.
- * @return A JSONObject which has only one field called "retValue" and the
- * value associated to it is the searched value. The methods of
- * JSONObject can be used to get the field value in a desired
- * format.
- * @throws JSONException
- * If the key is not found.
- */
- public JSONObject parse(String key) throws JSONException {
- initializeParser(key);
- parsing();
- return putResultInJSONObj();
- }
-
- /**
- * Prepare the fields of the class for the parsing
- *
- * @param key
- * The key whose value is searched for.
- * @throws JSONException
- * If the key is not found.
- */
- private void initializeParser(String key) throws JSONException {
- searchedfield = key;
- temp = new JSONObject(originalJO.toString());
- }
-
- /**
- * This function goes through the given field and calls the appropriate
- * functions to treat the units between the punctuation marks.
- *
- * @throws JSONException
- * If the key is not found.
- */
- private void parsing() throws JSONException {
- StringTokenizer st = new StringTokenizer(searchedfield, ".");
- while (st.hasMoreTokens()) {
- find(st.nextToken());
- }
- }
-
- /**
- * Search for the next part of the field and update the state if it was
- * found.
- *
- * @param nextToken
- * The current part of the searched field.
- * @throws JSONException
- * If the key is not found.
- */
- private void find(String nextToken) throws JSONException {
- if (endsWithBracket(nextToken)) {
- treatAllBracket(nextToken);
- } else {
- temp = ((JSONObject) temp).get(nextToken);
- }
- }
-
- /**
- * Determine whether the given string ends with a closing square bracket ']'
- *
- * @param nextToken
- * The current part of the searched field.
- * @return True if the given string ends with a closing square bracket ']'
- * and false otherwise.
- */
- private boolean endsWithBracket(String nextToken) {
- return nextToken.substring(nextToken.length() - 1).endsWith("]");
- }
-
- /**
- * Handle (multidimensional) arrays. Treat the square bracket pairs one
- * after the other if necessary.
- *
- * @param nextToken
- * The current part of the searched field.
- * @throws JSONException
- * If the searched element is not found.
- */
- private void treatAllBracket(String nextToken) throws JSONException {
- List<String> list = Arrays.asList(nextToken.split("\\["));
- ListIterator<String> iter = list.listIterator();
-
- temp = ((JSONObject) temp).get(iter.next());
-
- while (iter.hasNext()) {
- int index = Integer.parseInt(cutBracket(iter.next()));
- temp = ((JSONArray) temp).get(index);
- }
- }
-
- /**
- * Remove the last character of the string.
- *
- * @param string
- * String to modify.
- * @return The given string without the last character.
- */
- private String cutBracket(String string) {
- return string.substring(0, string.length() - 1);
- }
-
- /**
- * Save the result of the search into a JSONObject.
- *
- * @return A special JSONObject which contain only one key. The value
- * associated to this key is the result of the search.
- * @throws JSONException
- * If there is a problem creating the JSONObject. (e.g. invalid
- * syntax)
- */
- private JSONObject putResultInJSONObj() throws JSONException {
- JSONObject jo = new JSONObject();
- jo.put("retValue", temp);
- return jo;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
deleted file mode 100644
index 9bb87a0..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import java.util.Properties;
-
-import kafka.javaapi.producer.Producer;
-import kafka.producer.KeyedMessage;
-import kafka.producer.ProducerConfig;
-
-import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
-import org.apache.flink.streaming.connectors.util.SerializationSchema;
-
-public class KafkaSink<IN, OUT> extends RichSinkFunction<IN> {
- private static final long serialVersionUID = 1L;
-
- private kafka.javaapi.producer.Producer<Integer, OUT> producer;
- private Properties props;
- private String topicId;
- private String brokerAddr;
- private boolean initDone = false;
- private SerializationSchema<IN, OUT> scheme;
-
- public KafkaSink(String topicId, String brokerAddr,
- SerializationSchema<IN, OUT> serializationSchema) {
- this.topicId = topicId;
- this.brokerAddr = brokerAddr;
- this.scheme = serializationSchema;
-
- }
-
- /**
- * Initializes the connection to Kafka.
- */
- public void initialize() {
- props = new Properties();
-
- props.put("metadata.broker.list", brokerAddr);
- props.put("serializer.class", "kafka.serializer.StringEncoder");
- props.put("request.required.acks", "1");
-
- ProducerConfig config = new ProducerConfig(props);
- producer = new Producer<Integer, OUT>(config);
- initDone = true;
- }
-
- /**
- * Called when new data arrives to the sink, and forwards it to Kafka.
- *
- * @param next
- * The incoming data
- */
- @Override
- public void invoke(IN next) {
- if (!initDone) {
- initialize();
- }
-
- producer.send(new KeyedMessage<Integer, OUT>(topicId, scheme.serialize(next)));
-
- }
-
- @Override
- public void close() {
- producer.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
deleted file mode 100644
index 7328500..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import kafka.consumer.ConsumerConfig;
-import kafka.consumer.ConsumerIterator;
-import kafka.consumer.KafkaStream;
-import kafka.javaapi.consumer.ConsumerConnector;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.connectors.ConnectorSource;
-import org.apache.flink.streaming.connectors.util.DeserializationSchema;
-import org.apache.flink.util.Collector;
-
-public class KafkaSource<OUT> extends ConnectorSource<OUT> {
- private static final long serialVersionUID = 1L;
-
- private final String zkQuorum;
- private final String groupId;
- private final String topicId;
- private ConsumerConnector consumer;
-
- OUT outTuple;
-
- public KafkaSource(String zkQuorum, String groupId, String topicId,
- DeserializationSchema<OUT> deserializationSchema) {
- super(deserializationSchema);
- this.zkQuorum = zkQuorum;
- this.groupId = groupId;
- this.topicId = topicId;
- }
-
- /**
- * Initializes the connection to Kafka.
- */
- private void initializeConnection() {
- Properties props = new Properties();
- props.put("zookeeper.connect", zkQuorum);
- props.put("group.id", groupId);
- props.put("zookeeper.session.timeout.ms", "2000");
- props.put("zookeeper.sync.time.ms", "200");
- props.put("auto.commit.interval.ms", "1000");
- consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(props));
- }
-
- /**
- * Called to forward the data from the source to the {@link DataStream}.
- *
- * @param collector
- * The Collector for sending data to the dataStream
- */
- @Override
- public void invoke(Collector<OUT> collector) throws Exception {
-
- Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer
- .createMessageStreams(Collections.singletonMap(topicId, 1));
-
- KafkaStream<byte[], byte[]> stream = consumerMap.get(topicId).get(0);
- ConsumerIterator<byte[], byte[]> it = stream.iterator();
-
- while (it.hasNext()) {
- OUT out = schema.deserialize(it.next().message());
- if (schema.isEndOfStream(out)) {
- break;
- }
- collector.collect(out);
- }
- consumer.shutdown();
- }
-
- @Override
- public void open(Configuration config) {
- initializeConnection();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
deleted file mode 100644
index 7801d56..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.connectors.util.SimpleStringSchema;
-import org.apache.flink.util.Collector;
-
-public class KafkaTopology {
-
- public static final class MySource implements SourceFunction<String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(Collector<String> collector) throws Exception {
- for (int i = 0; i < 10; i++) {
- collector.collect(new String(Integer.toString(i)));
- }
- collector.collect(new String("q"));
-
- }
- }
-
- public static void main(String[] args) throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings("unused")
- DataStream<String> stream1 = env
- .addSource(
- new KafkaSource<String>("localhost:2181", "group", "test",
- new SimpleStringSchema())).print();
-
- @SuppressWarnings("unused")
- DataStream<String> stream2 = env.addSource(new MySource()).addSink(
- new KafkaSink<String, String>("test", "localhost:9092", new SimpleStringSchema()));
-
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
deleted file mode 100644
index 38c4f5f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.rabbitmq;
-
-import java.io.IOException;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
-import org.apache.flink.streaming.connectors.util.SerializationSchema;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
-
-public class RMQSink<IN> extends RichSinkFunction<IN> {
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(RMQSink.class);
-
- private String QUEUE_NAME;
- private String HOST_NAME;
- private transient ConnectionFactory factory;
- private transient Connection connection;
- private transient Channel channel;
- private SerializationSchema<IN, byte[]> scheme;
-
- public RMQSink(String HOST_NAME, String QUEUE_NAME, SerializationSchema<IN, byte[]> schema) {
- this.HOST_NAME = HOST_NAME;
- this.QUEUE_NAME = QUEUE_NAME;
- this.scheme = schema;
- }
-
- /**
- * Initializes the connection to RMQ.
- */
- public void initializeConnection() {
- factory = new ConnectionFactory();
- factory.setHost(HOST_NAME);
- try {
- connection = factory.newConnection();
- channel = connection.createChannel();
- channel.queueDeclare(QUEUE_NAME, false, false, false, null);
-
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Called when new data arrives to the sink, and forwards it to RMQ.
- *
- * @param value
- * The incoming data
- */
- @Override
- public void invoke(IN value) {
- try {
- byte[] msg = scheme.serialize(value);
-
- channel.basicPublish("", QUEUE_NAME, null, msg);
-
- } catch (IOException e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Cannot send RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
- }
- }
-
- }
-
- /**
- * Closes the connection.
- */
- private void closeChannel() {
- try {
- channel.close();
- connection.close();
- } catch (IOException e) {
- throw new RuntimeException("Error while closing RMQ connection with " + QUEUE_NAME
- + " at " + HOST_NAME, e);
- }
-
- }
-
- @Override
- public void open(Configuration config) {
- initializeConnection();
- }
-
- @Override
- public void close() {
- closeChannel();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
deleted file mode 100755
index 7ce864e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.rabbitmq;
-
-import java.io.IOException;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.connectors.ConnectorSource;
-import org.apache.flink.streaming.connectors.util.DeserializationSchema;
-import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
-import com.rabbitmq.client.QueueingConsumer;
-
-public class RMQSource<OUT> extends ConnectorSource<OUT> {
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(RMQSource.class);
-
- private final String QUEUE_NAME;
- private final String HOST_NAME;
-
- private transient ConnectionFactory factory;
- private transient Connection connection;
- private transient Channel channel;
- private transient QueueingConsumer consumer;
- private transient QueueingConsumer.Delivery delivery;
-
- OUT out;
-
- public RMQSource(String HOST_NAME, String QUEUE_NAME,
- DeserializationSchema<OUT> deserializationSchema) {
- super(deserializationSchema);
- this.HOST_NAME = HOST_NAME;
- this.QUEUE_NAME = QUEUE_NAME;
- }
-
- /**
- * Initializes the connection to RMQ.
- */
- private void initializeConnection() {
- factory = new ConnectionFactory();
- factory.setHost(HOST_NAME);
- try {
- connection = factory.newConnection();
- channel = connection.createChannel();
- channel.queueDeclare(QUEUE_NAME, false, false, false, null);
- consumer = new QueueingConsumer(channel);
- channel.basicConsume(QUEUE_NAME, true, consumer);
- } catch (IOException e) {
- throw new RuntimeException("Cannot create RMQ connection with " + QUEUE_NAME + " at "
- + HOST_NAME, e);
- }
- }
-
- /**
- * Called to forward the data from the source to the {@link DataStream}.
- *
- * @param collector
- * The Collector for sending data to the dataStream
- */
- @Override
- public void invoke(Collector<OUT> collector) throws Exception {
-
- while (true) {
-
- try {
- delivery = consumer.nextDelivery();
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Cannot recieve RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
- }
- }
-
- out = schema.deserialize(delivery.getBody());
- if (schema.isEndOfStream(out)) {
- break;
- } else {
- collector.collect(out);
- }
- }
-
- }
-
- @Override
- public void open(Configuration config) {
- initializeConnection();
- }
-
- @Override
- public void close() {
- try {
- connection.close();
- } catch (IOException e) {
- throw new RuntimeException("Error while closing RMQ connection with " + QUEUE_NAME
- + " at " + HOST_NAME, e);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
deleted file mode 100755
index a6ca9ae..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.rabbitmq;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.util.SerializationSchema;
-import org.apache.flink.streaming.connectors.util.SimpleStringSchema;
-
-public class RMQTopology {
-
- public static void main(String[] args) throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- @SuppressWarnings("unused")
- DataStream<String> dataStream1 = env.addSource(
- new RMQSource<String>("localhost", "hello", new SimpleStringSchema())).print();
-
- @SuppressWarnings("unused")
- DataStream<String> dataStream2 = env.fromElements("one", "two", "three", "four", "five",
- "q").addSink(
- new RMQSink<String>("localhost", "hello", new StringToByteSerializer()));
-
- env.execute();
- }
-
- public static class StringToByteSerializer implements SerializationSchema<String, byte[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public byte[] serialize(String element) {
- return element.getBytes();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
deleted file mode 100644
index ddb2538..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.twitter;
-
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.util.Properties;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.twitter.hbc.ClientBuilder;
-import com.twitter.hbc.core.Constants;
-import com.twitter.hbc.core.endpoint.StatusesSampleEndpoint;
-import com.twitter.hbc.core.processor.StringDelimitedProcessor;
-import com.twitter.hbc.httpclient.BasicClient;
-import com.twitter.hbc.httpclient.auth.Authentication;
-import com.twitter.hbc.httpclient.auth.OAuth1;
-
-/**
- * Implementation of {@link SourceFunction} specialized to emit tweets from
- * Twitter. It can connect to Twitter Streaming API, collect tweets and
- */
-public class TwitterSource extends RichParallelSourceFunction<String> {
-
- private static final Logger LOG = LoggerFactory.getLogger(TwitterSource.class);
-
- private static final long serialVersionUID = 1L;
- private String authPath;
- private transient BlockingQueue<String> queue;
- private int queueSize = 10000;
- private transient BasicClient client;
- private int waitSec = 5;
-
- private boolean streaming;
- private int numberOfTweets;
-
- /**
- * Create {@link TwitterSource} for streaming
- *
- * @param authPath
- * Location of the properties file containing the required
- * authentication information.
- */
- public TwitterSource(String authPath) {
- this.authPath = authPath;
- streaming = true;
- }
-
- /**
- * Create {@link TwitterSource} to collect finite number of tweets
- *
- * @param authPath
- * Location of the properties file containing the required
- * authentication information.
- * @param numberOfTweets
- *
- */
- public TwitterSource(String authPath, int numberOfTweets) {
- this.authPath = authPath;
- streaming = false;
- this.numberOfTweets = numberOfTweets;
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- initializeConnection();
- }
-
- @Override
- public void invoke(Collector<String> collector) throws Exception {
-
- if (streaming) {
- collectMessages(collector);
- } else {
- collectFiniteMessages(collector);
- }
- }
-
- @Override
- public void close() throws Exception {
- closeConnection();
- }
-
- /**
- * Initialize Hosebird Client to be able to consume Twitter's Streaming API
- */
- private void initializeConnection() {
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Initializing Twitter Streaming API connection");
- }
-
- queue = new LinkedBlockingQueue<String>(queueSize);
-
- StatusesSampleEndpoint endpoint = new StatusesSampleEndpoint();
- endpoint.stallWarnings(false);
-
- Authentication auth = authenticate();
-
- initializeClient(endpoint, auth);
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Twitter Streaming API connection established successfully");
- }
- }
-
- private OAuth1 authenticate() {
-
- Properties authenticationProperties = loadAuthenticationProperties();
-
- return new OAuth1(authenticationProperties.getProperty("consumerKey"),
- authenticationProperties.getProperty("consumerSecret"),
- authenticationProperties.getProperty("token"),
- authenticationProperties.getProperty("secret"));
- }
-
- /**
- * Reads the given properties file for the authentication data.
- *
- * @return the authentication data.
- */
- private Properties loadAuthenticationProperties() {
- Properties properties = new Properties();
- try {
- InputStream input = new FileInputStream(authPath);
- properties.load(input);
- input.close();
- } catch (Exception e) {
- throw new RuntimeException("Cannot open .properties file: " + authPath, e);
- }
- return properties;
- }
-
- private void initializeClient(StatusesSampleEndpoint endpoint, Authentication auth) {
-
- client = new ClientBuilder().name("twitterSourceClient").hosts(Constants.STREAM_HOST)
- .endpoint(endpoint).authentication(auth)
- .processor(new StringDelimitedProcessor(queue)).build();
-
- client.connect();
- }
-
- /**
- * Put tweets into collector
- *
- * @param collector
- * Collector in which the tweets are collected.
- */
- protected void collectFiniteMessages(Collector<String> collector) {
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Collecting tweets");
- }
-
- for (int i = 0; i < numberOfTweets; i++) {
- collectOneMessage(collector);
- }
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Collecting tweets finished");
- }
- }
-
- /**
- * Put tweets into collector
- *
- * @param collector
- * Collector in which the tweets are collected.
- */
- protected void collectMessages(Collector<String> collector) {
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Tweet-stream begins");
- }
-
- while (true) {
- collectOneMessage(collector);
- }
- }
-
- /**
- * Put one tweet into the collector.
- *
- * @param collector
- * Collector in which the tweets are collected.
- */
- protected void collectOneMessage(Collector<String> collector) {
- if (client.isDone()) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Client connection closed unexpectedly: {}", client.getExitEvent()
- .getMessage());
- }
- }
-
- try {
- String msg = queue.poll(waitSec, TimeUnit.SECONDS);
- if (msg != null) {
- collector.collect(msg);
- } else {
- if (LOG.isInfoEnabled()) {
- LOG.info("Did not receive a message in {} seconds", waitSec);
- }
- }
- } catch (InterruptedException e) {
- throw new RuntimeException("'Waiting for tweet' thread is interrupted", e);
- }
-
- }
-
- private void closeConnection() {
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Initiating connection close");
- }
-
- client.stop();
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Connection closed successfully");
- }
- }
-
- /**
- * Get the size of the queue in which the tweets are contained temporarily.
- *
- * @return the size of the queue in which the tweets are contained temporarily
- */
- public int getQueueSize() {
- return queueSize;
- }
-
- /**
- * Set the size of the queue in which the tweets are contained temporarily.
- *
- * @param queueSize
- * The desired value.
- */
- public void setQueueSize(int queueSize) {
- this.queueSize = queueSize;
- }
-
- /**
- * This function tells how long TwitterSource waits for the tweets.
- *
- * @return Number of second.
- */
- public int getWaitSec() {
- return waitSec;
- }
-
- /**
- * This function sets how long TwitterSource should wait for the tweets.
- *
- * @param waitSec
- * The desired value.
- */
- public void setWaitSec(int waitSec) {
- this.waitSec = waitSec;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
deleted file mode 100644
index a32fe1b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.twitter;
-
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
-import org.apache.flink.util.Collector;
-import org.apache.sling.commons.json.JSONException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TwitterStreaming {
-
- private static final int PARALLELISM = 1;
- private static final int SOURCE_PARALLELISM = 1;
- private static final int NUMBEROFTWEETS = 100;
-
- private static final Logger LOG = LoggerFactory.getLogger(TwitterStreaming.class);
-
- public static class TwitterSink implements SinkFunction<Tuple5<Long, Integer, String, String, String>> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(Tuple5<Long, Integer, String, String, String> tuple) {
- System.out.println("ID: " + tuple.f0 + " int: " + tuple.f1 + " LANGUAGE: " + tuple.f2);
- System.out.println("NAME: " + tuple.f4);
- System.out.println("TEXT: " + tuple.f3);
- System.out.println("");
- }
-
- }
-
- public static class SelectDataFlatMap extends
- JSONParseFlatMap<String, Tuple5<Long, Integer, String, String, String>> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(String value, Collector<Tuple5<Long, Integer, String, String, String>> out)
- throws Exception {
- try {
- out.collect(new Tuple5<Long, Integer, String, String, String>(
- getLong(value, "id"),
- getInt(value, "entities.hashtags[0].indices[1]"),
- getString(value, "lang"),
- getString(value, "text"),
- getString(value, "user.name")));
- } catch (JSONException e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Field not found");
- }
- }
- }
- }
-
- public static void main(String[] args) throws Exception {
-
- String path = new String();
-
- if (args != null && args.length == 1) {
- path = args[0];
- } else {
- System.err.println("USAGE:\nTwitterStreaming <pathToPropertiesFile>");
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment
- .createLocalEnvironment(PARALLELISM);
-
- DataStream<String> streamSource = env.addSource(new TwitterSource(path, NUMBEROFTWEETS))
- .setParallelism(SOURCE_PARALLELISM);
-
- DataStream<Tuple5<Long, Integer, String, String, String>> selectedDataStream = streamSource
- .flatMap(new SelectDataFlatMap());
-
- selectedDataStream.addSink(new TwitterSink());
-
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
deleted file mode 100644
index 4bc6df0..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.twitter;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
-import org.apache.flink.util.Collector;
-import org.apache.sling.commons.json.JSONException;
-
-/**
- * This program demonstrate the use of TwitterSource.
- * Its aim is to count the frequency of the languages of tweets
- */
-public class TwitterTopology {
-
- private static final int NUMBEROFTWEETS = 100;
-
- /**
- * FlatMapFunction to determine the language of tweets if possible
- */
- public static class SelectLanguageFlatMap extends
- JSONParseFlatMap<String, String> {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * Select the language from the incoming JSON text
- */
- @Override
- public void flatMap(String value, Collector<String> out) throws Exception {
- try{
- out.collect(getString(value, "lang"));
- }
- catch (JSONException e){
- out.collect("");
- }
- }
-
- }
-
- public static void main(String[] args) throws Exception {
-
- String path = new String();
-
- if (args != null && args.length == 1) {
- path = args[0];
- } else {
- System.err.println("USAGE:\nTwitterLocal <pathToPropertiesFile>");
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<String> streamSource = env.addSource(new TwitterSource(path, NUMBEROFTWEETS));
-
-
- DataStream<Tuple2<String, Integer>> dataStream = streamSource
- .flatMap(new SelectLanguageFlatMap())
- .map(new MapFunction<String, Tuple2<String, Integer>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<String, Integer> map(String value) throws Exception {
- return new Tuple2<String, Integer>(value, 1);
- }
- })
- .groupBy(0)
- .sum(1);
-
- dataStream.print();
-
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java
deleted file mode 100644
index 4507a1d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.util;
-
-import java.io.Serializable;
-
-public interface DeserializationSchema<T> extends Serializable {
-
- /**
- * Deserializes the incoming data.
- *
- * @param message
- * The incoming message in a byte array
- * @return The deserialized message in the required format.
- */
- public T deserialize(byte[] message);
-
- /**
- * Method to decide whether the element signals the end of the stream. If
- * true is returned the element won't be emitted
- *
- * @param nextElement
- * The element to test for end signal
- * @return The end signal, if true the stream shuts down
- */
- public boolean isEndOfStream(T nextElement);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java
deleted file mode 100644
index 29c749a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.util;
-
-public class RawSchema implements DeserializationSchema<byte[]>,
- SerializationSchema<byte[], byte[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public byte[] deserialize(byte[] message) {
- return message;
- }
-
- @Override
- public boolean isEndOfStream(byte[] nextElement) {
- return false;
- }
-
- @Override
- public byte[] serialize(byte[] element) {
- return element;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java
deleted file mode 100644
index 7c32312..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.util;
-
-import java.io.Serializable;
-
-public interface SerializationSchema<T,R> extends Serializable {
-
- /**
- * Serializes the incoming element to a specified type.
- *
- * @param element
- * The incoming element to be serialized
- * @return The serialized element.
- */
- public R serialize(T element);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java
deleted file mode 100644
index 4b21580..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.util;
-
-public class SimpleStringSchema implements DeserializationSchema<String>,
- SerializationSchema<String, String> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String deserialize(byte[] message) {
- return new String(message);
- }
-
- @Override
- public boolean isEndOfStream(String nextElement) {
- return false;
- }
-
- @Override
- public String serialize(String element) {
- return element;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/db/DBStateTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/db/DBStateTest.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/db/DBStateTest.java
deleted file mode 100644
index fe3783b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/db/DBStateTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.flink.streaming.connectors.db;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.HashMap;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-public class DBStateTest {
-
- public void stateTest(DBState<String, Integer> state) {
- state.put("k1", 1);
- assertEquals(Integer.valueOf(1), state.get("k1"));
-
- state.put("k2", 2);
- state.put("k3", 3);
- assertEquals(Integer.valueOf(2), state.get("k2"));
- assertEquals(Integer.valueOf(3), state.get("k3"));
- state.remove("k2");
-
- try {
- state.get("k2");
- fail();
- } catch (Exception e) {
- }
- }
-
- private void iteratorTest(DBStateWithIterator<String, Integer> state) {
- HashMap<String, Integer> expected = new HashMap<String, Integer>();
- HashMap<String, Integer> result = new HashMap<String, Integer>();
-
- state.put("10", 10);
- state.put("20", 20);
- state.put("30", 30);
-
- expected.put("10", 10);
- expected.put("20", 20);
- expected.put("30", 30);
-
- DBStateIterator<String, Integer> iterator = state.getIterator();
- while (iterator.hasNext()) {
- String key = iterator.getNextKey();
- Integer value = iterator.getNextValue();
- result.put(key, value);
- iterator.next();
- }
- state.close();
-
- assertEquals(expected, result);
- }
-
- // TODO
- @Ignore("Creates files with no licenses")
- @Test
- public void levelDBTest() {
- LevelDBState<String, Integer> state = new LevelDBState<String, Integer>("test");
- stateTest(state);
- state.close();
-
- state = new LevelDBState<String, Integer>("test");
- iteratorTest(state);
- state.close();
- }
-
- @Ignore("Needs running Memcached")
- @Test
- public void memcachedTest() {
- MemcachedState<Integer> state = new MemcachedState<Integer>();
- stateTest(state);
- state.close();
- }
-
- @Ignore("Needs running Redis")
- @Test
- public void redisTest() {
- RedisState<String, Integer> state = new RedisState<String, Integer>();
- stateTest(state);
- state.close();
-
- state = new RedisState<String, Integer>();
- iteratorTest(state);
- state.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
deleted file mode 100644
index b1d4115..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.json;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.flink.streaming.connectors.json.JSONParser;
-import org.apache.sling.commons.json.JSONException;
-import org.apache.sling.commons.json.JSONObject;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class JSONParserTest {
-
- private String jsonText;
- private String searchedField;
-
- public JSONParserTest(String text, String field) {
- jsonText = text;
- searchedField = field;
- }
-
- @Parameters
- public static Collection<Object[]> initParameterList() {
-
- Object[][] parameterList = new Object[][] {
- { "{\"key\":\"value\"}", "key" },
- { "{\"key\":[\"value\"]}", "key[0]" },
- { "{\"key\":[{\"key\":\"value\"}]}", "key[0].key" },
- { "{\"key\":[{\"key\":[{\"key\":\"value\"}]}]}", "key[0].key[0].key"},
- { "{\"key\":[1,[{\"key\":\"value\"}]]}", "key[1][0].key" },
- { "{\"key\":[1,[[\"key\",2,\"value\"]]]}", "key[1][0][2]" },
- { "{\"key\":{\"key\":{\"otherKey\":\"wrongValue\",\"key\":\"value\"},\"otherKey\":\"wrongValue\"},\"otherKey\":\"wrongValue\"}" , "key.key.key"}
- };
-
- return Arrays.asList(parameterList);
- }
-
- @Test
- public void test() {
- try {
- JSONParser parser = new JSONParser(jsonText);
- JSONObject jo = parser.parse(searchedField);
- String expected = "{\"retValue\":\"value\"}";
-
- assertTrue(expected.equals(jo.toString()));
- }
- catch (JSONException e) {
- fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
deleted file mode 100644
index 8851086..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.json;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.flink.streaming.connectors.json.JSONParser;
-import org.apache.sling.commons.json.JSONException;
-import org.apache.sling.commons.json.JSONObject;
-import org.junit.Test;
-
-
-public class JSONParserTest2 {
-
- @Test
- public void testGetBooleanFunction() {
- String jsonText = "{\"key\":true}";
- String searchedField = "key";
- try {
- JSONParser parser = new JSONParser(jsonText);
- JSONObject jo = parser.parse(searchedField);
-
- assertTrue(jo.getBoolean("retValue"));
- }
- catch (JSONException e) {
- fail();
- }
- }
-
- @Test
- public void testGetDoubleFunction() {
- double expected = 12345.12345;
- String jsonText = "{\"key\":" + expected + "}";
- String searchedField = "key";
- try {
- JSONParser parser = new JSONParser(jsonText);
- JSONObject jo = parser.parse(searchedField);
-
- assertEquals(expected,jo.getDouble("retValue"),0.000001);
- }
- catch (JSONException e) {
- fail();
- }
- }
-
- @Test
- public void testGetIntFunction() {
- int expected = 15;
- String jsonText = "{\"key\":" + expected + "}";
- String searchedField = "key";
- try {
- JSONParser parser = new JSONParser(jsonText);
- JSONObject jo = parser.parse(searchedField);
-
- assertEquals(expected,jo.getInt("retValue"));
- }
- catch (JSONException e) {
- fail();
- }
- }
-
- @Test
- public void testGetLongFunction() {
- long expected = 111111111111L;
- String jsonText = "{\"key\":" + expected + "}";
- String searchedField = "key";
- try {
- JSONParser parser = new JSONParser(jsonText);
- JSONObject jo = parser.parse(searchedField);
-
- assertEquals(expected,jo.getLong("retValue"));
- }
- catch (JSONException e) {
- fail();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties b/flink-addons/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
deleted file mode 100644
index 2fb9345..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
+++ /dev/null
@@ -1,19 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml b/flink-addons/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml
deleted file mode 100644
index 45b3b92..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
- </encoder>
- </appender>
-
- <root level="WARN">
- <appender-ref ref="STDOUT"/>
- </root>
- <logger name="org.apache.flink.streaming" level="WARN"/>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/pom.xml b/flink-addons/flink-streaming/flink-streaming-core/pom.xml
deleted file mode 100644
index a60bd01..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/pom.xml
+++ /dev/null
@@ -1,75 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-parent</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-streaming-core</artifactId>
- <name>flink-streaming-core</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math</artifactId>
- <version>2.2</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.sling</groupId>
- <artifactId>org.apache.sling.commons.json</artifactId>
- <version>2.0.6</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
[31/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java
deleted file mode 100644
index 0379fe0..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple10;
-import org.apache.flink.api.java.tuple.Tuple11;
-import org.apache.flink.api.java.tuple.Tuple12;
-import org.apache.flink.api.java.tuple.Tuple13;
-import org.apache.flink.api.java.tuple.Tuple14;
-import org.apache.flink.api.java.tuple.Tuple15;
-import org.apache.flink.api.java.tuple.Tuple16;
-import org.apache.flink.api.java.tuple.Tuple17;
-import org.apache.flink.api.java.tuple.Tuple18;
-import org.apache.flink.api.java.tuple.Tuple19;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple20;
-import org.apache.flink.api.java.tuple.Tuple21;
-import org.apache.flink.api.java.tuple.Tuple22;
-import org.apache.flink.api.java.tuple.Tuple23;
-import org.apache.flink.api.java.tuple.Tuple24;
-import org.apache.flink.api.java.tuple.Tuple25;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.api.java.tuple.Tuple6;
-import org.apache.flink.api.java.tuple.Tuple7;
-import org.apache.flink.api.java.tuple.Tuple8;
-import org.apache.flink.api.java.tuple.Tuple9;
-import org.apache.flink.streaming.api.windowing.extractor.FieldsFromTuple;
-import org.junit.Before;
-import org.junit.Test;
-
-public class FieldsFromTupleTest {
-
- private double[] testDouble;
-
- @Before
- public void init() {
- testDouble = new double[Tuple.MAX_ARITY];
- for (int i = 0; i < Tuple.MAX_ARITY; i++) {
- testDouble[i] = i;
- }
- }
-
- @Test
- public void testUserSpecifiedOrder() throws InstantiationException, IllegalAccessException {
- Tuple currentTuple = (Tuple) CLASSES[Tuple.MAX_ARITY - 1].newInstance();
- for (int i = 0; i < Tuple.MAX_ARITY; i++) {
- currentTuple.setField(testDouble[i], i);
- }
-
- double[] expected = { testDouble[5], testDouble[3], testDouble[6], testDouble[7],
- testDouble[0] };
- arrayEqualityCheck(expected, new FieldsFromTuple(5, 3, 6, 7, 0).extract(currentTuple));
-
- double[] expected2 = { testDouble[0], testDouble[Tuple.MAX_ARITY - 1] };
- arrayEqualityCheck(expected2,
- new FieldsFromTuple(0, Tuple.MAX_ARITY - 1).extract(currentTuple));
-
- double[] expected3 = { testDouble[Tuple.MAX_ARITY - 1], testDouble[0] };
- arrayEqualityCheck(expected3,
- new FieldsFromTuple(Tuple.MAX_ARITY - 1, 0).extract(currentTuple));
-
- double[] expected4 = { testDouble[13], testDouble[4], testDouble[5], testDouble[4],
- testDouble[2], testDouble[8], testDouble[6], testDouble[2], testDouble[8],
- testDouble[3], testDouble[5], testDouble[2], testDouble[16], testDouble[4],
- testDouble[3], testDouble[2], testDouble[6], testDouble[4], testDouble[7],
- testDouble[4], testDouble[2], testDouble[8], testDouble[7], testDouble[2] };
- arrayEqualityCheck(expected4, new FieldsFromTuple(13, 4, 5, 4, 2, 8, 6, 2, 8, 3, 5, 2, 16,
- 4, 3, 2, 6, 4, 7, 4, 2, 8, 7, 2).extract(currentTuple));
- }
-
- private void arrayEqualityCheck(double[] array1, double[] array2) {
- assertEquals("The result arrays must have the same length", array1.length, array2.length);
- for (int i = 0; i < array1.length; i++) {
- assertEquals("Unequal fields at position " + i, array1[i], array2[i], 0d);
- }
- }
-
- private static final Class<?>[] CLASSES = new Class<?>[] { Tuple1.class, Tuple2.class,
- Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
- Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
- Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
- Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
- Tuple24.class, Tuple25.class };
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java
deleted file mode 100644
index 4773f4e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-public class CountEvictionPolicyTest {
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- @Test
- public void testCountEvictionPolicy() {
- List<Integer> tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
- int counter;
-
- // The count policy should not care about the triggered parameter
- // Therefore its value switches after each use in this test.
- boolean triggered = false;
- // the size of the buffer should not matter as well!
-
- // Test count of different sizes (0..9)
- for (int i = 0; i < 10; i++) {
- EvictionPolicy evictionPolicy = new CountEvictionPolicy(i, i);
- counter = 0;
-
- // Test first i steps (should not evict)
- for (int j = 0; j < i; j++) {
- counter++;
- assertEquals("Evictionpolicy with count of " + i + " evicted tuples at add nr. "
- + counter + ". It should not evict for the first " + i + " adds.", 0,
- evictionPolicy.notifyEviction(tuples.get(j), (triggered = !triggered),
- tuples.get(Math.abs((i - j)) % 10)));
- }
-
- // Test the next three evictions
- for (int j = 0; j < 3; j++) {
- // The first add should evict now
- counter++;
- assertEquals("Evictionpolicy with count of " + i
- + " did not evict correct number of tuples at the expected pos " + counter
- + ".", i, evictionPolicy.notifyEviction(tuples.get(j),
- (triggered = !triggered), tuples.get(Math.abs((i - j)) % 10)));
-
- // the next i-1 adds should not evict
- for (int k = 0; k < i - 1; k++) {
- counter++;
- assertEquals("Evictionpolicy with count of " + i
- + " evicted tuples at add nr. " + counter, 0,
- evictionPolicy.notifyEviction(tuples.get(j), (triggered = !triggered),
- tuples.get(Math.abs((i - j)) % 10)));
- }
- }
- }
- }
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- @Test
- public void testCountEvictionPolicyStartValuesAndEvictionAmount() {
-
- // The count policy should not care about the triggered parameter
- // Therefore its value switches after each use in this test.
- boolean triggered = false;
- // the size of the buffer should not matter as well!
-
- List<Integer> tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-
- // Text different eviction amounts (0..3)
- for (int x = 0; x < 4; x++) {
-
- // Test count of different sizes (0..9)
- for (int i = 0; i < 10; i++) {
-
- int counter = 0;
-
- // Test different start values (-5..5)
- for (int j = -5; i < 6; i++) {
- EvictionPolicy evictionPolicy = new CountEvictionPolicy(i, x, j);
- // Add tuples without eviction
- for (int k = 0; k < ((i - j > 0) ? i - j : 0); k++) {
- counter++;
- assertEquals("Evictionpolicy with count of " + i
- + " did not evict correct number of tuples at the expected pos "
- + counter + ".", 0, evictionPolicy.notifyEviction(
- tuples.get(Math.abs(j)), (triggered = !triggered),
- tuples.get(Math.abs((i - j)) % 10)));
- }
- // Expect eviction
- counter++;
- assertEquals("Evictionpolicy with count of " + i
- + " did not evict correct number of tuples at the expected pos "
- + counter + ".", x, evictionPolicy.notifyEviction(
- tuples.get(Math.abs(j)), (triggered = !triggered),
- tuples.get(Math.abs((i - j)) % 10)));
- }
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java
deleted file mode 100644
index b7120e7..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.api.windowing.helper.Count;
-import org.junit.Test;
-
-public class CountTriggerPolicyTest {
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- @Test
- public void testCountTriggerPolicy() {
-
- List tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
- int counter;
-
- // Test count of different sizes (0..9)
- for (int i = 0; i < 10; i++) {
- TriggerPolicy triggerPolicy = Count.of(i).toTrigger();
- counter=0;
-
- // Test first i steps (should not trigger)
- for (int j = 0; j < i; j++) {
- counter++;
- assertFalse("Triggerpolicy with count of " + i + " triggered at add nr. " + counter
- + ". It should not trigger for the first " + i + " adds.",
- triggerPolicy.notifyTrigger(tuples.get(j)));
- }
-
- // Test the next three triggers
- for (int j = 0; j < 3; j++) {
- // The first add should trigger now
- counter++;
- assertTrue("Triggerpolicy with count of " + i
- + " did not trigger at the expected pos " + counter + ".",
- triggerPolicy.notifyTrigger(tuples.get(j)));
-
- // the next i-1 adds should not trigger
- for (int k = 0; k < i - 1; k++) {
- counter++;
- assertFalse("Triggerpolicy with count of " + i + " triggered at add nr. "
- + counter, triggerPolicy.notifyTrigger(tuples.get(k)));
- }
- }
- }
- }
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- @Test
- public void testCountTriggerPolicyStartValues() {
-
- List tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-
- // Test count of different sizes (0..9)
- for (int i = 0; i < 10; i++) {
-
- // Test different start values (-5..5)
- for (int j = -5; i < 6; i++) {
- TriggerPolicy triggerPolicy = new CountTriggerPolicy(i, j);
- // Add tuples without trigger
- for (int k = 0; k < ((i - j > 0) ? i - j : 0); k++) {
- assertFalse("Triggerpolicy with count of " + i + " and start value of " + j
- + " triggered at add nr. " + (k + 1),
- triggerPolicy.notifyTrigger(tuples.get(k % 10)));
- }
- // Expect trigger
- assertTrue("Triggerpolicy with count of " + i + "and start value of " + j
- + " did not trigger at the expected position.",
- triggerPolicy.notifyTrigger(tuples.get(0)));
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java
deleted file mode 100644
index 424eb57..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
-import org.apache.flink.streaming.api.windowing.policy.DeltaPolicy;
-import org.junit.Test;
-
-import java.util.List;
-import java.util.Arrays;
-
-import static org.junit.Assert.*;
-
-public class DeltaPolicyTest {
-
- @SuppressWarnings({ "serial", "unchecked", "rawtypes" })
- @Test
- public void testDelta() {
- DeltaPolicy deltaPolicy = new DeltaPolicy(new DeltaFunction<Tuple2<Integer, Integer>>() {
- @Override
- public double getDelta(Tuple2<Integer, Integer> oldDataPoint, Tuple2<Integer, Integer> newDataPoint) {
- return (double) newDataPoint.f0 - oldDataPoint.f0;
- }
- }, new Tuple2(0, 0), 2);
-
- List<Tuple2> tuples = Arrays.asList(
- new Tuple2(1, 0),
- new Tuple2(2, 0),
- new Tuple2(3, 0),
- new Tuple2(6, 0));
-
- assertFalse(deltaPolicy.notifyTrigger(tuples.get(0)));
- assertEquals(0, deltaPolicy.notifyEviction(tuples.get(0), false, 0));
-
- assertFalse(deltaPolicy.notifyTrigger(tuples.get(1)));
- assertEquals(0, deltaPolicy.notifyEviction(tuples.get(1), false, 1));
-
- assertTrue(deltaPolicy.notifyTrigger(tuples.get(2)));
- assertEquals(1, deltaPolicy.notifyEviction(tuples.get(2), true, 2));
-
- assertTrue(deltaPolicy.notifyTrigger(tuples.get(3)));
- assertEquals(2, deltaPolicy.notifyEviction(tuples.get(3), true, 2));
- }
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java
deleted file mode 100644
index 8224b12..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.windowing.extractor.FieldFromTuple;
-import org.junit.Test;
-
-public class PunctuationPolicyTest {
-
- // This value should not effect the policy. It is changed at each call to
- // verify this.
- private boolean triggered = false;
-
- @Test
- public void PunctuationTriggerTestWithoutExtraction() {
- PunctuationPolicy<Object, Object> policy = new PunctuationPolicy<Object, Object>(
- new TestObject(0));
- assertTrue("The present punctuation was not detected. (POS 1)",
- policy.notifyTrigger(new TestObject(0)));
- assertFalse("There was a punctuation detected which wasn't present. (POS 2)",
- policy.notifyTrigger(new TestObject(1)));
- }
-
- @Test
- public void PunctuationTriggerTestWithExtraction() {
- @SuppressWarnings({ "unchecked", "rawtypes" })
- PunctuationPolicy<Tuple2<Object, Object>, Object> policy = new PunctuationPolicy<Tuple2<Object, Object>, Object>(
- new TestObject(0), new FieldFromTuple(0));
- assertTrue("The present punctuation was not detected. (POS 3)",
- policy.notifyTrigger(new Tuple2<Object, Object>(new TestObject(0),
- new TestObject(1))));
- assertFalse("There was a punctuation detected which wasn't present. (POS 4)",
- policy.notifyTrigger(new Tuple2<Object, Object>(new TestObject(1),
- new TestObject(0))));
- }
-
- @Test
- public void PunctuationEvictionTestWithoutExtraction() {
- // The current buffer size should not effect the test. It's therefore
- // always 0 here.
-
- PunctuationPolicy<Object, Object> policy = new PunctuationPolicy<Object, Object>(
- new TestObject(0));
- assertEquals(
- "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 5)",
- 0, policy.notifyEviction(new TestObject(0), (triggered = !triggered), 0));
- for (int i = 0; i < 10; i++) {
- for (int j = 0; j < i; j++) {
- assertEquals("There was a punctuation detected which wasn't present. (POS 6)", 0,
- policy.notifyEviction(new TestObject(1), (triggered = !triggered), 0));
- }
- assertEquals(
- "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 7)",
- i + 1, policy.notifyEviction(new TestObject(0), (triggered = !triggered), 0));
- }
- }
-
- @Test
- public void PunctuationEvictionTestWithExtraction() {
- // The current buffer size should not effect the test. It's therefore
- // always 0 here.
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- PunctuationPolicy<Tuple2<Object, Object>, Object> policy = new PunctuationPolicy<Tuple2<Object, Object>, Object>(
- new TestObject(0), new FieldFromTuple(0));
- assertEquals(
- "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 10)",
- 0, policy.notifyEviction(new Tuple2<Object, Object>(new TestObject(0),
- new TestObject(1)), (triggered = !triggered), 0));
- for (int i = 0; i < 10; i++) {
- for (int j = 0; j < i; j++) {
- assertEquals("There was a punctuation detected which wasn't present. (POS 9)", 0,
- policy.notifyEviction(new Tuple2<Object, Object>(new TestObject(1),
- new TestObject(0)), (triggered = !triggered), 0));
- }
- assertEquals(
- "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 10)",
- i + 1, policy.notifyEviction(new Tuple2<Object, Object>(new TestObject(0),
- new TestObject(1)), (triggered = !triggered), 0));
- }
- }
-
- private class TestObject {
-
- private int id;
-
- public TestObject(int id) {
- this.id = id;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof TestObject && ((TestObject) o).getId() == this.id) {
- return true;
- } else {
- return false;
- }
- }
-
- public int getId() {
- return id;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
deleted file mode 100644
index b5d502b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import java.util.LinkedList;
-
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-public class TimeEvictionPolicyTest {
-
- @Test
- public void timeEvictionTest() {
- // create some test data
- Integer[] times = { 1, 3, 4, 6, 7, 9, 14, 20, 21, 22, 30, 31, 33, 36, 40, 41, 42, 43, 44,
- 45, 47, 55 };
- Integer[] numToDelete = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 3 };
-
- // create a timestamp
- @SuppressWarnings("serial")
- Timestamp<Integer> timeStamp = new Timestamp<Integer>() {
-
- @Override
- public long getTimestamp(Integer value) {
- return value;
- }
-
- };
-
- // test different granularity
- for (long granularity = 0; granularity < 40; granularity++) {
- // create policy
- TimeEvictionPolicy<Integer> policy = new TimeEvictionPolicy<Integer>(granularity,
- new TimestampWrapper<Integer>(timeStamp, 0));
-
- // The trigger status should not effect the policy. Therefore, it's
- // value is changed after each usage.
- boolean triggered = false;
-
- // The eviction should work similar with both, fake and real
- // elements. Which kind is used is changed on every 3rd element in
- // this test.
- int fakeAndRealCounter = 0;
- boolean fake = false;
-
- // test by adding values
- LinkedList<Integer> buffer = new LinkedList<Integer>();
- for (int i = 0; i < times.length; i++) {
-
- // check if the current element should be a fake
- fakeAndRealCounter++;
- if (fakeAndRealCounter > 2) {
- fake = !fake;
- fakeAndRealCounter = 0;
- }
-
- int result;
-
- if (fake) {
- // Notify eviction with fake element
- result = policy.notifyEvictionWithFakeElement(times[i], buffer.size());
- } else {
- // Notify eviction with real element
- result = policy.notifyEviction(times[i], (triggered = !triggered),
- buffer.size());
- }
-
- // handle correctness of eviction
- for (; result > 0 && !buffer.isEmpty(); result--) {
- if (buffer.getFirst() <= times[i] - granularity) {
- buffer.removeFirst();
- } else {
- fail("The policy wanted to evict time " + buffer.getFirst()
- + " while the current time was " + times[i]
- + "and the granularity was " + granularity);
- }
- }
-
- // test that all required evictions have been done
- if (!buffer.isEmpty()) {
- assertTrue("The policy did not evict " + buffer.getFirst()
- + " while the current time was " + times[i]
- + " and the granularity was " + granularity,
- (buffer.getFirst() >= times[i] - granularity));
- }
-
- // test influence of other evictions
- for (int j = numToDelete[i % numToDelete.length]; j > 0; j--) {
- if (!buffer.isEmpty()) {
- buffer.removeFirst();
- }
- }
-
- // add current element to buffer if it is no fake
- if (!fake) {
- buffer.add(times[i]);
- }
-
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
deleted file mode 100644
index 2bdbd96..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.junit.Test;
-
-public class TimeTriggerPolicyTest {
-
- @Test
- public void timeTriggerRegularNotifyTest() {
- // create some test data
- Integer[] times = { 1, 3, 4, 6, 7, 9, 14, 20, 21, 22, 30 };
-
- // create a timestamp
- @SuppressWarnings("serial")
- Timestamp<Integer> timeStamp = new Timestamp<Integer>() {
-
- @Override
- public long getTimestamp(Integer value) {
- return value;
- }
-
- };
-
- // test different granularity
- for (long granularity = 0; granularity < 31; granularity++) {
- // create policy
-
- TriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(granularity,
- new TimestampWrapper<Integer>(timeStamp, 0));
-
- // remember window border
- long currentTime = 0;
-
- // test by adding values
- for (int i = 0; i < times.length; i++) {
- boolean result = policy.notifyTrigger(times[i]);
- // start time is included, but end time is excluded: >=
- if (times[i] >= currentTime + granularity) {
- if (granularity != 0) {
- currentTime = times[i] - ((times[i] - currentTime) % granularity);
- }
- assertTrue("The policy did not trigger at pos " + i + " (current time border: "
- + currentTime + "; current granularity: " + granularity
- + "; data point time: " + times[i] + ")", result);
- } else {
- assertFalse("The policy triggered wrong at pos " + i
- + " (current time border: " + currentTime + "; current granularity: "
- + granularity + "; data point time: " + times[i] + ")", result);
- }
- }
- }
-
- }
-
- @Test
- public void timeTriggerPreNotifyTest() {
- // create some test data
- Integer[] times = { 1, 3, 20, 26 };
-
- // create a timestamp
- @SuppressWarnings("serial")
- Timestamp<Integer> timeStamp = new Timestamp<Integer>() {
-
- @Override
- public long getTimestamp(Integer value) {
- return value;
- }
-
- };
-
- // create policy
- TimeTriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(5,
- new TimestampWrapper<Integer>(timeStamp, 0));
-
- // expected result
- Long[][] result = { {}, {}, { 4L, 9L, 14L, 19L }, { 24L } };
-
- // call policy
- for (int i = 0; i < times.length; i++) {
- arrayEqualityCheck(result[i], policy.preNotifyTrigger(times[i]));
- policy.notifyTrigger(times[i]);
- }
- }
-
- private void arrayEqualityCheck(Object[] array1, Object[] array2) {
- assertEquals("The result arrays must have the same length", array1.length, array2.length);
- for (int i = 0; i < array1.length; i++) {
- assertEquals("Unequal fields at position " + i, array1[i], array2[i]);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java
deleted file mode 100644
index 8cb03a7..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import static org.junit.Assert.*;
-
-import org.junit.Test;
-
-public class TumblingEvictionPolicyTest {
-
- @Test
- public void testTumblingEviction() {
- EvictionPolicy<Integer> policy = new TumblingEvictionPolicy<Integer>();
-
- int counter = 0;
-
- for (int i = 0; i < 10; i++) {
- for (int j = 0; j < i; j++) {
- assertEquals(0, policy.notifyEviction(0, false, counter++));
- }
- assertEquals(counter, policy.notifyEviction(0, true, counter));
- counter = 1;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/BroadcastPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/BroadcastPartitionerTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/BroadcastPartitionerTest.java
deleted file mode 100644
index de6c200..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/BroadcastPartitionerTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import static org.junit.Assert.assertArrayEquals;
-
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.junit.Before;
-import org.junit.Test;
-
-public class BroadcastPartitionerTest {
-
- private BroadcastPartitioner<Tuple> broadcastPartitioner1;
- private BroadcastPartitioner<Tuple> broadcastPartitioner2;
- private BroadcastPartitioner<Tuple> broadcastPartitioner3;
-
- private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
- private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(null);
-
- @Before
- public void setPartitioner() {
- broadcastPartitioner1 = new BroadcastPartitioner<Tuple>();
- broadcastPartitioner2 = new BroadcastPartitioner<Tuple>();
- broadcastPartitioner3 = new BroadcastPartitioner<Tuple>();
-
- }
-
- @Test
- public void testSelectChannels() {
- int[] first = new int[] { 0 };
- int[] second = new int[] { 0, 1 };
- int[] sixth = new int[] { 0, 1, 2, 3, 4, 5 };
- sd.setInstance(streamRecord);
- assertArrayEquals(first, broadcastPartitioner1.selectChannels(sd, 1));
- assertArrayEquals(second, broadcastPartitioner2.selectChannels(sd, 2));
- assertArrayEquals(sixth, broadcastPartitioner3.selectChannels(sd, 6));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java
deleted file mode 100644
index 0675242..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.junit.Before;
-import org.junit.Test;
-
-public class DistributePartitionerTest {
-
- private DistributePartitioner<Tuple> distributePartitioner;
- private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
- private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
- null);
-
- @Before
- public void setPartitioner() {
- distributePartitioner = new DistributePartitioner<Tuple>(false);
- }
-
- @Test
- public void testSelectChannelsLength() {
- sd.setInstance(streamRecord);
- assertEquals(1, distributePartitioner.selectChannels(sd, 1).length);
- assertEquals(1, distributePartitioner.selectChannels(sd, 2).length);
- assertEquals(1, distributePartitioner.selectChannels(sd, 1024).length);
- }
-
- @Test
- public void testSelectChannelsInterval() {
- sd.setInstance(streamRecord);
- assertEquals(0, distributePartitioner.selectChannels(sd, 3)[0]);
- assertEquals(1, distributePartitioner.selectChannels(sd, 3)[0]);
- assertEquals(2, distributePartitioner.selectChannels(sd, 3)[0]);
- assertEquals(0, distributePartitioner.selectChannels(sd, 3)[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java
deleted file mode 100644
index b56649b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.junit.Before;
-import org.junit.Test;
-
-public class FieldsPartitionerTest {
-
- private FieldsPartitioner<Tuple> fieldsPartitioner;
- private StreamRecord<Tuple> streamRecord1 = new StreamRecord<Tuple>()
- .setObject(new Tuple2<String, Integer>("test", 0));
- private StreamRecord<Tuple> streamRecord2 = new StreamRecord<Tuple>()
- .setObject(new Tuple2<String, Integer>("test", 42));
- private SerializationDelegate<StreamRecord<Tuple>> sd1 = new SerializationDelegate<StreamRecord<Tuple>>(
- null);
- private SerializationDelegate<StreamRecord<Tuple>> sd2 = new SerializationDelegate<StreamRecord<Tuple>>(
- null);
-
- @Before
- public void setPartitioner() {
- fieldsPartitioner = new FieldsPartitioner<Tuple>(new KeySelector<Tuple, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String getKey(Tuple value) throws Exception {
- return value.getField(0);
- }
- });
- }
-
- @Test
- public void testSelectChannelsLength() {
- sd1.setInstance(streamRecord1);
- assertEquals(1, fieldsPartitioner.selectChannels(sd1, 1).length);
- assertEquals(1, fieldsPartitioner.selectChannels(sd1, 2).length);
- assertEquals(1, fieldsPartitioner.selectChannels(sd1, 1024).length);
- }
-
- @Test
- public void testSelectChannelsGrouping() {
- sd1.setInstance(streamRecord1);
- sd2.setInstance(streamRecord2);
-
- assertArrayEquals(fieldsPartitioner.selectChannels(sd1, 1),
- fieldsPartitioner.selectChannels(sd2, 1));
- assertArrayEquals(fieldsPartitioner.selectChannels(sd1, 2),
- fieldsPartitioner.selectChannels(sd2, 2));
- assertArrayEquals(fieldsPartitioner.selectChannels(sd1, 1024),
- fieldsPartitioner.selectChannels(sd2, 1024));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java
deleted file mode 100644
index b381d85..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ForwardPartitionerTest {
-
- private DistributePartitioner<Tuple> forwardPartitioner;
- private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
- private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
- null);
-
- @Before
- public void setPartitioner() {
- forwardPartitioner = new DistributePartitioner<Tuple>(true);
- }
-
- @Test
- public void testSelectChannelsLength() {
- sd.setInstance(streamRecord);
- assertEquals(1, forwardPartitioner.selectChannels(sd, 1).length);
- assertEquals(1, forwardPartitioner.selectChannels(sd, 2).length);
- assertEquals(1, forwardPartitioner.selectChannels(sd, 1024).length);
- }
-
- @Test
- public void testSelectChannelsInterval() {
- sd.setInstance(streamRecord);
- assertEquals(0, forwardPartitioner.selectChannels(sd, 1)[0]);
- assertEquals(1, forwardPartitioner.selectChannels(sd, 2)[0]);
- assertEquals(2, forwardPartitioner.selectChannels(sd, 1024)[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/GlobalPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/GlobalPartitionerTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/GlobalPartitionerTest.java
deleted file mode 100644
index eebda64..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/GlobalPartitionerTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import static org.junit.Assert.assertArrayEquals;
-
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.junit.Before;
-import org.junit.Test;
-
-public class GlobalPartitionerTest {
-
- private GlobalPartitioner<Tuple> globalPartitioner;
- private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
- private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
- null);
-
- @Before
- public void setPartitioner() {
- globalPartitioner = new GlobalPartitioner<Tuple>();
- }
-
- @Test
- public void testSelectChannels() {
- int[] result = new int[] { 0 };
-
- sd.setInstance(streamRecord);
-
- assertArrayEquals(result, globalPartitioner.selectChannels(sd, 1));
- assertArrayEquals(result, globalPartitioner.selectChannels(sd, 2));
- assertArrayEquals(result, globalPartitioner.selectChannels(sd, 1024));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ShufflePartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ShufflePartitionerTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ShufflePartitionerTest.java
deleted file mode 100644
index 3c03d07..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ShufflePartitionerTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ShufflePartitionerTest {
-
- private ShufflePartitioner<Tuple> shufflePartitioner;
- private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
- private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
- null);
-
- @Before
- public void setPartitioner() {
- shufflePartitioner = new ShufflePartitioner<Tuple>();
- }
-
- @Test
- public void testSelectChannelsLength() {
- sd.setInstance(streamRecord);
- assertEquals(1, shufflePartitioner.selectChannels(sd, 1).length);
- assertEquals(1, shufflePartitioner.selectChannels(sd, 2).length);
- assertEquals(1, shufflePartitioner.selectChannels(sd, 1024).length);
- }
-
- @Test
- public void testSelectChannelsInterval() {
- sd.setInstance(streamRecord);
- assertEquals(0, shufflePartitioner.selectChannels(sd, 1)[0]);
-
- assertTrue(0 <= shufflePartitioner.selectChannels(sd, 2)[0]);
- assertTrue(2 > shufflePartitioner.selectChannels(sd, 2)[0]);
-
- assertTrue(0 <= shufflePartitioner.selectChannels(sd, 1024)[0]);
- assertTrue(1024 > shufflePartitioner.selectChannels(sd, 1024)[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/MapStateTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/MapStateTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/MapStateTest.java
deleted file mode 100644
index 194403c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/MapStateTest.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.streaming.state.checkpoint.MapCheckpoint;
-import org.apache.flink.streaming.state.checkpoint.StateCheckpoint;
-import org.junit.Test;
-
-public class MapStateTest {
-
- @Test
- public void testMapState() {
-
- Map<String, Integer> map = new HashMap<String, Integer>();
- map.put("a", 1);
- map.put("b", 2);
- map.put("c", 3);
- map.remove("a");
-
- MapState<String, Integer> mapState = new MapState<String, Integer>();
- mapState.put("a", 1);
- mapState.put("b", 2);
- mapState.put("c", 3);
-
- assertEquals(1, (int) mapState.remove("a"));
- assertEquals(null, mapState.remove("a"));
- assertEquals(2, mapState.size());
- assertEquals(map, mapState.state);
- assertEquals(map.entrySet(), mapState.entrySet());
- assertTrue(mapState.containsKey("b"));
- assertFalse(mapState.containsKey("a"));
-
- assertEquals(2, mapState.updatedItems.size());
- assertEquals(1, mapState.removedItems.size());
-
- Map<String, Integer> map2 = new HashMap<String, Integer>();
- map2.put("a", 0);
- map2.put("e", -1);
-
- mapState.putAll(map2);
-
- assertEquals(4, mapState.updatedItems.size());
- assertEquals(0, mapState.removedItems.size());
-
- mapState.clear();
- assertEquals(new HashMap<String, Integer>(), mapState.state);
- assertTrue(mapState.clear);
- assertEquals(0, mapState.updatedItems.size());
- assertEquals(0, mapState.removedItems.size());
-
- mapState.putAll(map);
- assertEquals(map.keySet(), mapState.updatedItems);
-
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testMapStateCheckpointing() {
-
- Map<String, Integer> map = new HashMap<String, Integer>();
- map.put("a", 1);
- map.put("b", 2);
- map.put("c", 3);
-
- MapState<String, Integer> mapState = new MapState<String, Integer>();
- mapState.putAll(map);
-
- StateCheckpoint<Map<String, Integer>> mcp = mapState.checkpoint();
- assertEquals(map, mcp.getCheckpointedState());
-
- Map<String, Integer> map2 = new HashMap<String, Integer>();
- map2.put("a", 0);
- map2.put("e", -1);
-
- mapState.put("a", 0);
- mapState.put("e", -1);
- mapState.remove("b");
- StateCheckpoint<Map<String, Integer>> mcp2 = new MapCheckpoint<String, Integer>(mapState);
- assertEquals(map2, mcp2.getCheckpointedState());
- mcp.update(mcp2);
- assertEquals(mapState.state, mcp.getCheckpointedState());
-
- mapState.clear();
- mapState.put("a", 1);
- mapState.put("a", 2);
- mapState.put("b", -3);
- mapState.put("c", 0);
- mapState.remove("b");
- mcp.update(mapState.checkpoint());
- assertEquals(mapState.state, mcp.getCheckpointedState());
-
- MapState<String, Integer> mapState2 = (MapState<String, Integer>) new MapState<String, Integer>()
- .restore(mcp);
-
- assertTrue(mapState2.stateEquals(mapState));
- mapState2.reBuild(mapState2.repartition(10));
- assertTrue(mapState2.stateEquals(mapState));
-
- MapState<Integer, Integer> mapState3 = new MapState<Integer, Integer>();
- mapState3.put(1, 1);
- mapState3.put(2, 1);
-
- mapState3.reBuild(mapState3.repartition(2)[0]);
- assertTrue(mapState3.containsKey(2));
- assertFalse(mapState3.containsKey(1));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/OperatorStateTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/OperatorStateTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/OperatorStateTest.java
deleted file mode 100644
index 6cb8f51..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/OperatorStateTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.flink.streaming.state.checkpoint.StateCheckpoint;
-import org.junit.Test;
-
-public class OperatorStateTest {
-
- @Test
- public void testOperatorState() {
- OperatorState<Integer> os = new SimpleState<Integer>(5);
-
- StateCheckpoint<Integer> scp = os.checkpoint();
-
- assertTrue(os.stateEquals(new SimpleState<Integer>().restore(scp)));
-
- assertEquals((Integer) 5, os.getState());
-
- os.setState(10);
-
- assertEquals((Integer) 10, os.getState());
-
- scp.update(os.checkpoint());
-
- assertEquals((Integer)10,scp.getCheckpointedState());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
deleted file mode 100644
index ea94f98..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.StreamConfig;
-import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.api.streamvertex.StreamTaskContext;
-import org.apache.flink.streaming.io.CoReaderIterator;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.MutableObjectIterator;
-
-public class MockCoContext<IN1, IN2, OUT> implements StreamTaskContext<OUT> {
- // private Collection<IN1> input1;
- // private Collection<IN2> input2;
- private Iterator<IN1> inputIterator1;
- private Iterator<IN2> inputIterator2;
- private List<OUT> outputs;
-
- private Collector<OUT> collector;
- private StreamRecordSerializer<IN1> inDeserializer1;
- private CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>> mockIterator;
- private StreamRecordSerializer<IN2> inDeserializer2;
-
- public MockCoContext(Collection<IN1> input1, Collection<IN2> input2) {
-
- if (input1.isEmpty() || input2.isEmpty()) {
- throw new RuntimeException("Inputs must not be empty");
- }
-
- this.inputIterator1 = input1.iterator();
- this.inputIterator2 = input2.iterator();
-
- TypeInformation<IN1> inTypeInfo1 = TypeExtractor.getForObject(input1.iterator().next());
- inDeserializer1 = new StreamRecordSerializer<IN1>(inTypeInfo1);
- TypeInformation<IN2> inTypeInfo2 = TypeExtractor.getForObject(input2.iterator().next());
- inDeserializer2 = new StreamRecordSerializer<IN2>(inTypeInfo2);
-
- mockIterator = new MockCoReaderIterator(inDeserializer1, inDeserializer2);
-
- outputs = new ArrayList<OUT>();
- collector = new MockCollector<OUT>(outputs);
- }
-
- private int currentInput = 1;
- private StreamRecord<IN1> reuse1;
- private StreamRecord<IN2> reuse2;
-
- private class MockCoReaderIterator extends
- CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>> {
-
- public MockCoReaderIterator(TypeSerializer<StreamRecord<IN1>> serializer1,
- TypeSerializer<StreamRecord<IN2>> serializer2) {
- super(null, serializer1, serializer2);
- reuse1 = inDeserializer1.createInstance();
- reuse2 = inDeserializer2.createInstance();
- }
-
- @Override
- public int next(StreamRecord<IN1> target1, StreamRecord<IN2> target2) throws IOException {
- this.delegate1.setInstance(target1);
- this.delegate2.setInstance(target2);
-
- int inputNumber = nextRecord();
- target1.setObject(reuse1.getObject());
- target2.setObject(reuse2.getObject());
-
- return inputNumber;
- }
- }
-
- private Integer nextRecord() {
- if (inputIterator1.hasNext() && inputIterator2.hasNext()) {
- switch (currentInput) {
- case 1:
- return next1();
- case 2:
- return next2();
- default:
- return 0;
- }
- }
-
- if (inputIterator1.hasNext()) {
- return next1();
- }
-
- if (inputIterator2.hasNext()) {
- return next2();
- }
-
- return 0;
- }
-
- private int next1() {
- reuse1 = inDeserializer1.createInstance();
- reuse1.setObject(inputIterator1.next());
- currentInput = 2;
- return 1;
- }
-
- private int next2() {
- reuse2 = inDeserializer2.createInstance();
- reuse2.setObject(inputIterator2.next());
- currentInput = 1;
- return 2;
- }
-
- public List<OUT> getOutputs() {
- return outputs;
- }
-
- public Collector<OUT> getCollector() {
- return collector;
- }
-
- public StreamRecordSerializer<IN1> getInDeserializer1() {
- return inDeserializer1;
- }
-
- public StreamRecordSerializer<IN2> getInDeserializer2() {
- return inDeserializer2;
- }
-
- public CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>> getIterator() {
- return mockIterator;
- }
-
- public static <IN1, IN2, OUT> List<OUT> createAndExecute(CoInvokable<IN1, IN2, OUT> invokable,
- List<IN1> input1, List<IN2> input2) {
- MockCoContext<IN1, IN2, OUT> mockContext = new MockCoContext<IN1, IN2, OUT>(input1, input2);
- invokable.setup(mockContext);
-
- try {
- invokable.open(null);
- invokable.invoke();
- invokable.close();
- } catch (Exception e) {
- throw new RuntimeException("Cannot invoke invokable.", e);
- }
-
- return mockContext.getOutputs();
- }
-
- @Override
- public StreamConfig getConfig() {
- return null;
- }
-
- @Override
- public ClassLoader getUserCodeClassLoader() {
- return null;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <X> MutableObjectIterator<X> getInput(int index) {
- switch (index) {
- case 0:
- return (MutableObjectIterator<X>) inputIterator1;
- case 1:
- return (MutableObjectIterator<X>) inputIterator2;
- default:
- throw new IllegalArgumentException("CoStreamVertex has only 2 inputs");
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <X> StreamRecordSerializer<X> getInputSerializer(int index) {
- switch (index) {
- case 0:
- return (StreamRecordSerializer<X>) inDeserializer1;
- case 1:
- return (StreamRecordSerializer<X>) inDeserializer2;
- default:
- throw new IllegalArgumentException("CoStreamVertex has only 2 inputs");
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <X, Y> CoReaderIterator<X, Y> getCoReader() {
- return (CoReaderIterator<X, Y>) mockIterator;
- }
-
- @Override
- public Collector<OUT> getOutputCollector() {
- return collector;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCollector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCollector.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCollector.java
deleted file mode 100644
index e8b96c5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCollector.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import java.io.Serializable;
-import java.util.Collection;
-
-import org.apache.commons.lang3.SerializationUtils;
-import org.apache.flink.util.Collector;
-
-public class MockCollector<T> implements Collector<T> {
- private Collection<T> outputs;
-
- public MockCollector(Collection<T> outputs) {
- this.outputs = outputs;
- }
-
- @Override
- public void collect(T record) {
- T copied = SerializationUtils.deserialize(SerializationUtils
- .serialize((Serializable) record));
- outputs.add(copied);
- }
-
- @Override
- public void close() {
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
deleted file mode 100644
index 5537052..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.StreamConfig;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.api.streamvertex.StreamTaskContext;
-import org.apache.flink.streaming.io.CoReaderIterator;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.MutableObjectIterator;
-
-public class MockContext<IN, OUT> implements StreamTaskContext<OUT> {
- private Collection<IN> inputs;
- private List<OUT> outputs;
-
- private Collector<OUT> collector;
- private StreamRecordSerializer<IN> inDeserializer;
- private MutableObjectIterator<StreamRecord<IN>> iterator;
-
- public MockContext(Collection<IN> inputs) {
- this.inputs = inputs;
- if (inputs.isEmpty()) {
- throw new RuntimeException("Inputs must not be empty");
- }
-
- TypeInformation<IN> inTypeInfo = TypeExtractor.getForObject(inputs.iterator().next());
- inDeserializer = new StreamRecordSerializer<IN>(inTypeInfo);
-
- iterator = new MockInputIterator();
- outputs = new ArrayList<OUT>();
- collector = new MockCollector<OUT>(outputs);
- }
-
- private class MockInputIterator implements MutableObjectIterator<StreamRecord<IN>> {
- Iterator<IN> listIterator;
-
- public MockInputIterator() {
- listIterator = inputs.iterator();
- }
-
- @Override
- public StreamRecord<IN> next(StreamRecord<IN> reuse) throws IOException {
- if (listIterator.hasNext()) {
- reuse.setObject(listIterator.next());
- } else {
- reuse = null;
- }
- return reuse;
- }
-
- @Override
- public StreamRecord<IN> next() throws IOException {
- if (listIterator.hasNext()) {
- StreamRecord<IN> result = new StreamRecord<IN>();
- result.setObject(listIterator.next());
- return result;
- } else {
- return null;
- }
- }
- }
-
- public List<OUT> getOutputs() {
- return outputs;
- }
-
- public Collector<OUT> getCollector() {
- return collector;
- }
-
- public StreamRecordSerializer<IN> getInDeserializer() {
- return inDeserializer;
- }
-
- public MutableObjectIterator<StreamRecord<IN>> getIterator() {
- return iterator;
- }
-
- public static <IN, OUT> List<OUT> createAndExecute(StreamInvokable<IN, OUT> invokable,
- List<IN> inputs) {
- MockContext<IN, OUT> mockContext = new MockContext<IN, OUT>(inputs);
- invokable.setup(mockContext);
- try {
- invokable.open(null);
- invokable.invoke();
- invokable.close();
- } catch (Exception e) {
- throw new RuntimeException("Cannot invoke invokable.", e);
- }
-
- return mockContext.getOutputs();
- }
-
- @Override
- public StreamConfig getConfig() {
- return null;
- }
-
- @Override
- public ClassLoader getUserCodeClassLoader() {
- return null;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <X> MutableObjectIterator<X> getInput(int index) {
- if (index == 0) {
- return (MutableObjectIterator<X>) iterator;
- } else {
- throw new IllegalArgumentException("There is only 1 input");
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <X> StreamRecordSerializer<X> getInputSerializer(int index) {
- if (index == 0) {
- return (StreamRecordSerializer<X>) inDeserializer;
- } else {
- throw new IllegalArgumentException("There is only 1 input");
- }
- }
-
- @Override
- public Collector<OUT> getOutputCollector() {
- return collector;
- }
-
- @Override
- public <X, Y> CoReaderIterator<X, Y> getCoReader() {
- throw new IllegalArgumentException("CoReader not available");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockRecordWriterFactory.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockRecordWriterFactory.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockRecordWriterFactory.java
deleted file mode 100644
index 88673a3..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockRecordWriterFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import static org.mockito.Mockito.doCallRealMethod;
-import static org.mockito.Mockito.mock;
-
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamvertex.MockRecordWriter;
-import org.mockito.Mockito;
-
-public class MockRecordWriterFactory {
-
- @SuppressWarnings("unchecked")
- public static MockRecordWriter create() {
- MockRecordWriter recWriter = mock(MockRecordWriter.class);
-
- Mockito.when(recWriter.initList()).thenCallRealMethod();
- doCallRealMethod().when(recWriter).emit(Mockito.any(SerializationDelegate.class));
-
- recWriter.initList();
-
- return recWriter;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockSource.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockSource.java
deleted file mode 100644
index bb92e8e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockSource.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-
-public class MockSource<T> {
-
- public static <T> List<T> createAndExecute(SourceFunction<T> source) {
- List<T> outputs = new ArrayList<T>();
- try {
- source.invoke(new MockCollector<T>(outputs));
- } catch (Exception e) {
- throw new RuntimeException("Cannot invoke source.", e);
- }
- return outputs;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
deleted file mode 100644
index 95a5d9b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import akka.actor.ActorRef;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.client.JobClient;
-import org.apache.flink.runtime.client.JobExecutionException;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.test.util.ForkableFlinkMiniCluster;
-
-public class TestStreamEnvironment extends StreamExecutionEnvironment {
- private static final String DEFAULT_JOBNAME = "TestStreamingJob";
- private static final String CANNOT_EXECUTE_EMPTY_JOB = "Cannot execute empty job";
-
- private long memorySize;
-
- public TestStreamEnvironment(int degreeOfParallelism, long memorySize){
- this.setDegreeOfParallelism(degreeOfParallelism);
-
- this.memorySize = memorySize;
- }
-
- @Override
- public void execute() throws Exception {
- execute(DEFAULT_JOBNAME);
- }
-
- @Override
- public void execute(String jobName) throws Exception {
- JobGraph jobGraph = streamGraph.getJobGraph(jobName);
-
- Configuration configuration = jobGraph.getJobConfiguration();
-
- configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS,
- getDegreeOfParallelism());
- configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, memorySize);
-
- ForkableFlinkMiniCluster cluster = new ForkableFlinkMiniCluster(configuration);
-
- try{
- ActorRef client = cluster.getJobClient();
- JobClient.submitJobAndWait(jobGraph, false, client, cluster.timeout());
- }catch(JobExecutionException e){
- if(e.getMessage().contains("GraphConversionException")){
- throw new Exception(CANNOT_EXECUTE_EMPTY_JOB, e);
- }else{
- throw e;
- }
- }finally{
- cluster.stop();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties b/flink-addons/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties
deleted file mode 100644
index 2fb9345..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties
+++ /dev/null
@@ -1,19 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/resources/logback-test.xml b/flink-addons/flink-streaming/flink-streaming-core/src/test/resources/logback-test.xml
deleted file mode 100644
index 4f56748..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
- </encoder>
- </appender>
-
- <root level="WARN">
- <appender-ref ref="STDOUT"/>
- </root>
- <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
-</configuration>
\ No newline at end of file
[16/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
new file mode 100644
index 0000000..aed4496
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -0,0 +1,1271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.Validate;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.RichFilterFunction;
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.functions.RichReduceFunction;
+import org.apache.flink.api.common.io.OutputFormat;
+import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.ClosureCleaner;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.io.CsvOutputFormat;
+import org.apache.flink.api.java.io.TextOutputFormat;
+import org.apache.flink.api.java.operators.Keys;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.core.fs.FileSystem.WriteMode;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.streaming.api.StreamGraph;
+import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.datastream.temporaloperator.StreamCrossOperator;
+import org.apache.flink.streaming.api.datastream.temporaloperator.StreamJoinOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
+import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
+import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
+import org.apache.flink.streaming.api.function.sink.FileSinkFunctionByMillis;
+import org.apache.flink.streaming.api.function.sink.PrintSinkFunction;
+import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.api.invokable.SinkInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.operator.CounterInvokable;
+import org.apache.flink.streaming.api.invokable.operator.FilterInvokable;
+import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable;
+import org.apache.flink.streaming.api.invokable.operator.MapInvokable;
+import org.apache.flink.streaming.api.invokable.operator.StreamReduceInvokable;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.streaming.api.windowing.helper.Delta;
+import org.apache.flink.streaming.api.windowing.helper.Time;
+import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.partitioner.BroadcastPartitioner;
+import org.apache.flink.streaming.partitioner.DistributePartitioner;
+import org.apache.flink.streaming.partitioner.FieldsPartitioner;
+import org.apache.flink.streaming.partitioner.GlobalPartitioner;
+import org.apache.flink.streaming.partitioner.ShufflePartitioner;
+import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.streaming.util.keys.KeySelectorUtil;
+
+/**
+ * A DataStream represents a stream of elements of the same type. A DataStream
+ * can be transformed into another DataStream by applying a transformation as
+ * for example
+ * <ul>
+ * <li>{@link DataStream#map},</li>
+ * <li>{@link DataStream#filter}, or</li>
+ * <li>{@link DataStream#aggregate}.</li>
+ * </ul>
+ *
+ * @param <OUT>
+ * The type of the DataStream, i.e., the type of the elements of the
+ * DataStream.
+ */
+public class DataStream<OUT> {
+
+ protected static Integer counter = 0;
+ protected final StreamExecutionEnvironment environment;
+ protected final String id;
+ protected final String type;
+ protected int degreeOfParallelism;
+ protected List<String> userDefinedNames;
+ protected StreamPartitioner<OUT> partitioner;
+ @SuppressWarnings("rawtypes")
+ protected TypeInformation typeInfo;
+ protected List<DataStream<OUT>> mergedStreams;
+
+ protected final StreamGraph streamGraph;
+
+ /**
+ * Create a new {@link DataStream} in the given execution environment with
+ * partitioning set to forward by default.
+ *
+ * @param environment
+ * StreamExecutionEnvironment
+ * @param operatorType
+ * The type of the operator in the component
+ * @param typeInfo
+ * Type of the datastream
+ */
+ public DataStream(StreamExecutionEnvironment environment, String operatorType,
+ TypeInformation<OUT> typeInfo) {
+ if (environment == null) {
+ throw new NullPointerException("context is null");
+ }
+
+ counter++;
+ this.id = counter.toString();
+ this.type = operatorType;
+ this.environment = environment;
+ this.degreeOfParallelism = environment.getDegreeOfParallelism();
+ this.streamGraph = environment.getStreamGraph();
+ this.userDefinedNames = new ArrayList<String>();
+ this.partitioner = new DistributePartitioner<OUT>(true);
+ this.typeInfo = typeInfo;
+ this.mergedStreams = new ArrayList<DataStream<OUT>>();
+ this.mergedStreams.add(this);
+ }
+
+ /**
+ * Create a new DataStream by creating a copy of another DataStream
+ *
+ * @param dataStream
+ * The DataStream that will be copied.
+ */
+ public DataStream(DataStream<OUT> dataStream) {
+ this.environment = dataStream.environment;
+ this.id = dataStream.id;
+ this.type = dataStream.type;
+ this.degreeOfParallelism = dataStream.degreeOfParallelism;
+ this.userDefinedNames = new ArrayList<String>(dataStream.userDefinedNames);
+ this.partitioner = dataStream.partitioner;
+ this.streamGraph = dataStream.streamGraph;
+ this.typeInfo = dataStream.typeInfo;
+ this.mergedStreams = new ArrayList<DataStream<OUT>>();
+ this.mergedStreams.add(this);
+ if (dataStream.mergedStreams.size() > 1) {
+ for (int i = 1; i < dataStream.mergedStreams.size(); i++) {
+ this.mergedStreams.add(new DataStream<OUT>(dataStream.mergedStreams.get(i)));
+ }
+ }
+
+ }
+
+ /**
+ * Returns the ID of the {@link DataStream}.
+ *
+ * @return ID of the DataStream
+ */
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * Gets the degree of parallelism for this operator.
+ *
+ * @return The parallelism set for this operator.
+ */
+ public int getParallelism() {
+ return this.degreeOfParallelism;
+ }
+
+ /**
+ * Gets the type of the stream.
+ *
+ * @return The type of the datastream.
+ */
+ @SuppressWarnings("unchecked")
+ public TypeInformation<OUT> getType() {
+ return this.typeInfo;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <R> DataStream<R> setType(TypeInformation<R> outType) {
+ streamGraph.setOutType(id, outType);
+ typeInfo = outType;
+ return (DataStream<R>) this;
+ }
+
+ public <F> F clean(F f) {
+ if (getExecutionEnvironment().getConfig().isClosureCleanerEnabled()) {
+ ClosureCleaner.clean(f, true);
+ }
+ ClosureCleaner.ensureSerializable(f);
+ return f;
+ }
+
+ public StreamExecutionEnvironment getExecutionEnvironment() {
+ return environment;
+ }
+
+ /**
+ * Creates a new {@link DataStream} by merging {@link DataStream} outputs of
+ * the same type with each other. The DataStreams merged using this operator
+ * will be transformed simultaneously.
+ *
+ * @param streams
+ * The DataStreams to merge output with.
+ * @return The {@link DataStream}.
+ */
+ public DataStream<OUT> merge(DataStream<OUT>... streams) {
+ DataStream<OUT> returnStream = this.copy();
+
+ for (DataStream<OUT> stream : streams) {
+ for (DataStream<OUT> ds : stream.mergedStreams) {
+ validateMerge(ds.getId());
+ returnStream.mergedStreams.add(ds.copy());
+ }
+ }
+ return returnStream;
+ }
+
+ /**
+ * Operator used for directing tuples to specific named outputs using an
+ * {@link org.apache.flink.streaming.api.collector.OutputSelector}. Calling
+ * this method on an operator creates a new {@link SplitDataStream}.
+ *
+ * @param outputSelector
+ * The user defined
+ * {@link org.apache.flink.streaming.api.collector.OutputSelector}
+ * for directing the tuples.
+ * @return The {@link SplitDataStream}
+ */
+ public SplitDataStream<OUT> split(OutputSelector<OUT> outputSelector) {
+ for (DataStream<OUT> ds : this.mergedStreams) {
+ streamGraph.setOutputSelector(ds.getId(), clean(outputSelector));
+ }
+
+ return new SplitDataStream<OUT>(this);
+ }
+
+ /**
+ * Creates a new {@link ConnectedDataStream} by connecting
+ * {@link DataStream} outputs of different type with each other. The
+ * DataStreams connected using this operators can be used with CoFunctions.
+ *
+ * @param dataStream
+ * The DataStream with which this stream will be joined.
+ * @return The {@link ConnectedDataStream}.
+ */
+ public <R> ConnectedDataStream<OUT, R> connect(DataStream<R> dataStream) {
+ return new ConnectedDataStream<OUT, R>(this, dataStream);
+ }
+
+ /**
+ * Groups the elements of a {@link DataStream} by the given key positions to
+ * be used with grouped operators like
+ * {@link GroupedDataStream#reduce(ReduceFunction)}</p> This operator also
+ * affects the partitioning of the stream, by forcing values with the same
+ * key to go to the same processing instance.
+ *
+ * @param fields
+ * The position of the fields on which the {@link DataStream}
+ * will be grouped.
+ * @return The grouped {@link DataStream}
+ */
+ public GroupedDataStream<OUT> groupBy(int... fields) {
+ if (getType() instanceof BasicArrayTypeInfo || getType() instanceof PrimitiveArrayTypeInfo) {
+ return groupBy(new KeySelectorUtil.ArrayKeySelector<OUT>(fields));
+ } else {
+ return groupBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
+ }
+ }
+
+ /**
+ * Groups a {@link DataStream} using field expressions. A field expression
+ * is either the name of a public field or a getter method with parentheses
+ * of the {@link DataStream}S underlying type. A dot can be used to drill
+ * down into objects, as in {@code "field1.getInnerField2()" }. This method
+ * returns an {@link GroupedDataStream}.</p> This operator also affects the
+ * partitioning of the stream, by forcing values with the same key to go to
+ * the same processing instance.
+ *
+ * @param fields
+ * One or more field expressions on which the DataStream will be
+ * grouped.
+ * @return The grouped {@link DataStream}
+ **/
+ public GroupedDataStream<OUT> groupBy(String... fields) {
+
+ return groupBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
+
+ }
+
+ /**
+ * Groups the elements of a {@link DataStream} by the key extracted by the
+ * {@link KeySelector} to be used with grouped operators like
+ * {@link GroupedDataStream#reduce(ReduceFunction)}.
+ * <p/>
+ * This operator also affects the partitioning of the stream, by forcing
+ * values with the same key to go to the same processing instance.
+ *
+ * @param keySelector
+ * The {@link KeySelector} that will be used to extract keys for
+ * the values
+ * @return The grouped {@link DataStream}
+ */
+ public GroupedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
+ return new GroupedDataStream<OUT>(this, clean(keySelector));
+ }
+
+ private GroupedDataStream<OUT> groupBy(Keys<OUT> keys) {
+ return new GroupedDataStream<OUT>(this, clean(KeySelectorUtil.getSelectorForKeys(keys,
+ getType())));
+ }
+
+ /**
+ * Sets the partitioning of the {@link DataStream} so that the output is
+ * partitioned using the given {@link KeySelector}. This setting only
+ * effects the how the outputs will be distributed between the parallel
+ * instances of the next processing operator.
+ *
+ * @param keySelector
+ * @return
+ */
+ protected DataStream<OUT> partitionBy(KeySelector<OUT, ?> keySelector) {
+ return setConnectionType(new FieldsPartitioner<OUT>(clean(keySelector)));
+ }
+
+ /**
+ * Sets the partitioning of the {@link DataStream} so that the output tuples
+ * are broadcasted to every parallel instance of the next component. This
+ * setting only effects the how the outputs will be distributed between the
+ * parallel instances of the next processing operator.
+ *
+ * @return The DataStream with broadcast partitioning set.
+ */
+ public DataStream<OUT> broadcast() {
+ return setConnectionType(new BroadcastPartitioner<OUT>());
+ }
+
+ /**
+ * Sets the partitioning of the {@link DataStream} so that the output tuples
+ * are shuffled to the next component. This setting only effects the how the
+ * outputs will be distributed between the parallel instances of the next
+ * processing operator.
+ *
+ * @return The DataStream with shuffle partitioning set.
+ */
+ public DataStream<OUT> shuffle() {
+ return setConnectionType(new ShufflePartitioner<OUT>());
+ }
+
+ /**
+ * Sets the partitioning of the {@link DataStream} so that the output tuples
+ * are forwarded to the local subtask of the next component (whenever
+ * possible). This is the default partitioner setting. This setting only
+ * effects the how the outputs will be distributed between the parallel
+ * instances of the next processing operator.
+ *
+ * @return The DataStream with shuffle partitioning set.
+ */
+ public DataStream<OUT> forward() {
+ return setConnectionType(new DistributePartitioner<OUT>(true));
+ }
+
+ /**
+ * Sets the partitioning of the {@link DataStream} so that the output tuples
+ * are distributed evenly to the next component.This setting only effects
+ * the how the outputs will be distributed between the parallel instances of
+ * the next processing operator.
+ *
+ * @return The DataStream with shuffle partitioning set.
+ */
+ public DataStream<OUT> distribute() {
+ return setConnectionType(new DistributePartitioner<OUT>(false));
+ }
+
+ /**
+ * Sets the partitioning of the {@link DataStream} so that the output values
+ * all go to the first instance of the next processing operator. Use this
+ * setting with care since it might cause a serious performance bottleneck
+ * in the application.
+ *
+ * @return The DataStream with shuffle partitioning set.
+ */
+ public DataStream<OUT> global() {
+ return setConnectionType(new GlobalPartitioner<OUT>());
+ }
+
+ /**
+ * Initiates an iterative part of the program that feeds back data streams.
+ * The iterative part needs to be closed by calling
+ * {@link IterativeDataStream#closeWith(DataStream)}. The transformation of
+ * this IterativeDataStream will be the iteration head. The data stream
+ * given to the {@link IterativeDataStream#closeWith(DataStream)} method is
+ * the data stream that will be fed back and used as the input for the
+ * iteration head. A common usage pattern for streaming iterations is to use
+ * output splitting to send a part of the closing data stream to the head.
+ * Refer to {@link #split(OutputSelector)} for more information.
+ * <p>
+ * The iteration edge will be partitioned the same way as the first input of
+ * the iteration head.
+ * <p>
+ * By default a DataStream with iteration will never terminate, but the user
+ * can use the maxWaitTime parameter to set a max waiting time for the
+ * iteration head. If no data received in the set time, the stream
+ * terminates.
+ *
+ * @return The iterative data stream created.
+ */
+ public IterativeDataStream<OUT> iterate() {
+ return new IterativeDataStream<OUT>(this, 0);
+ }
+
+ /**
+ * Initiates an iterative part of the program that feeds back data streams.
+ * The iterative part needs to be closed by calling
+ * {@link IterativeDataStream#closeWith(DataStream)}. The transformation of
+ * this IterativeDataStream will be the iteration head. The data stream
+ * given to the {@link IterativeDataStream#closeWith(DataStream)} method is
+ * the data stream that will be fed back and used as the input for the
+ * iteration head. A common usage pattern for streaming iterations is to use
+ * output splitting to send a part of the closing data stream to the head.
+ * Refer to {@link #split(OutputSelector)} for more information.
+ * <p>
+ * The iteration edge will be partitioned the same way as the first input of
+ * the iteration head.
+ * <p>
+ * By default a DataStream with iteration will never terminate, but the user
+ * can use the maxWaitTime parameter to set a max waiting time for the
+ * iteration head. If no data received in the set time, the stream
+ * terminates.
+ *
+ * @param maxWaitTimeMillis
+ * Number of milliseconds to wait between inputs before shutting
+ * down
+ *
+ * @return The iterative data stream created.
+ */
+ public IterativeDataStream<OUT> iterate(long maxWaitTimeMillis) {
+ return new IterativeDataStream<OUT>(this, maxWaitTimeMillis);
+ }
+
+ /**
+ * Applies a Map transformation on a {@link DataStream}. The transformation
+ * calls a {@link MapFunction} for each element of the DataStream. Each
+ * MapFunction call returns exactly one element. The user can also extend
+ * {@link RichMapFunction} to gain access to other features provided by the
+ * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ *
+ * @param mapper
+ * The MapFunction that is called for each element of the
+ * DataStream.
+ * @param <R>
+ * output type
+ * @return The transformed {@link DataStream}.
+ */
+ public <R> SingleOutputStreamOperator<R, ?> map(MapFunction<OUT, R> mapper) {
+
+ TypeInformation<R> outType = TypeExtractor.getMapReturnTypes(clean(mapper), getType());
+
+ return transform("Map", outType, new MapInvokable<OUT, R>(clean(mapper)));
+ }
+
+ /**
+ * Applies a FlatMap transformation on a {@link DataStream}. The
+ * transformation calls a {@link FlatMapFunction} for each element of the
+ * DataStream. Each FlatMapFunction call can return any number of elements
+ * including none. The user can also extend {@link RichFlatMapFunction} to
+ * gain access to other features provided by the
+ * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ *
+ * @param flatMapper
+ * The FlatMapFunction that is called for each element of the
+ * DataStream
+ *
+ * @param <R>
+ * output type
+ * @return The transformed {@link DataStream}.
+ */
+ public <R> SingleOutputStreamOperator<R, ?> flatMap(FlatMapFunction<OUT, R> flatMapper) {
+
+ TypeInformation<R> outType = TypeExtractor.getFlatMapReturnTypes(clean(flatMapper),
+ getType());
+
+ return transform("Flat Map", outType, new FlatMapInvokable<OUT, R>(clean(flatMapper)));
+
+ }
+
+ /**
+ * Applies a reduce transformation on the data stream. The user can also
+ * extend the {@link RichReduceFunction} to gain access to other features
+ * provided by the
+ * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ *
+ * @param reducer
+ * The {@link ReduceFunction} that will be called for every
+ * element of the input values.
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reducer) {
+
+ return transform("Reduce", getType(), new StreamReduceInvokable<OUT>(clean(reducer)));
+
+ }
+
+ /**
+ * Applies a Filter transformation on a {@link DataStream}. The
+ * transformation calls a {@link FilterFunction} for each element of the
+ * DataStream and retains only those element for which the function returns
+ * true. Elements for which the function returns false are filtered. The
+ * user can also extend {@link RichFilterFunction} to gain access to other
+ * features provided by the
+ * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ *
+ * @param filter
+ * The FilterFunction that is called for each element of the
+ * DataSet.
+ * @return The filtered DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> filter(FilterFunction<OUT> filter) {
+ return transform("Filter", getType(), new FilterInvokable<OUT>(clean(filter)));
+
+ }
+
+ /**
+ * Initiates a Project transformation on a {@link Tuple} {@link DataStream}.<br/>
+ * <b>Note: Only Tuple DataStreams can be projected.</b></br> The
+ * transformation projects each Tuple of the DataSet onto a (sub)set of
+ * fields.</br> This method returns a {@link StreamProjection} on which
+ * {@link StreamProjection#types(Class)} needs to be called to completed the
+ * transformation.
+ *
+ * @param fieldIndexes
+ * The field indexes of the input tuples that are retained. The
+ * order of fields in the output tuple corresponds to the order
+ * of field indexes.
+ * @return A StreamProjection that needs to be converted into a DataStream
+ * to complete the project transformation by calling
+ * {@link StreamProjection#types(Class)}.
+ *
+ * @see Tuple
+ * @see DataStream
+ */
+ public StreamProjection<OUT> project(int... fieldIndexes) {
+ return new StreamProjection<OUT>(this.copy(), fieldIndexes);
+ }
+
+ /**
+ * Initiates a temporal Cross transformation.<br/>
+ * A Cross transformation combines the elements of two {@link DataStream}s
+ * into one DataStream over a specified time window. It builds all pair
+ * combinations of elements of both DataStreams, i.e., it builds a Cartesian
+ * product.
+ *
+ * <p>
+ * This method returns a {@link StreamCrossOperator} on which the
+ * {@link StreamCrossOperator#onWindow} should be called to define the
+ * window.
+ * <p>
+ * Call {@link StreamCrossOperator.CrossWindow#with(crossFunction)} to
+ * define a custom cross function.
+ *
+ * @param dataStreamToCross
+ * The other DataStream with which this DataStream is crossed.
+ * @return A {@link StreamCrossOperator} to continue the definition of the
+ * cross transformation.
+ *
+ */
+ public <IN2> StreamCrossOperator<OUT, IN2> cross(DataStream<IN2> dataStreamToCross) {
+ return new StreamCrossOperator<OUT, IN2>(this, dataStreamToCross);
+ }
+
+ /**
+ * Initiates a temporal Join transformation. <br/>
+ * A temporal Join transformation joins the elements of two
+ * {@link DataStream}s on key equality over a specified time window.</br>
+ *
+ * This method returns a {@link StreamJoinOperator} on which the
+ * {@link StreamJoinOperator#onWindow} should be called to define the
+ * window, and then the {@link StreamJoinOperator.JoinWindow#where} and
+ * {@link StreamJoinOperator.JoinPredicate#equalTo} can be used to define
+ * the join keys.</p> The user can also use the
+ * {@link StreamJoinOperator.JoinedStream#with(joinFunction)} to apply
+ * custom join function.
+ *
+ * @param other
+ * The other DataStream with which this DataStream is joined.
+ * @return A {@link StreamJoinOperator} to continue the definition of the
+ * Join transformation.
+ *
+ */
+ public <IN2> StreamJoinOperator<OUT, IN2> join(DataStream<IN2> dataStreamToJoin) {
+ return new StreamJoinOperator<OUT, IN2>(this, dataStreamToJoin);
+ }
+
+ /**
+ * Applies an aggregation that sums the data stream at the given position.
+ *
+ * @param positionToSum
+ * The position in the data point to sum
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> sum(int positionToSum) {
+ checkFieldRange(positionToSum);
+ return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(positionToSum,
+ getClassAtPos(positionToSum), getType()));
+ }
+
+ /**
+ * Applies an aggregation that that gives the current sum of the pojo data
+ * stream at the given field expression. A field expression is either the
+ * name of a public field or a getter method with parentheses of the
+ * {@link DataStream}S underlying type. A dot can be used to drill down into
+ * objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> sum(String field) {
+ return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field, getType()));
+ }
+
+ /**
+ * Applies an aggregation that that gives the current minimum of the data
+ * stream at the given position.
+ *
+ * @param positionToMin
+ * The position in the data point to minimize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> min(int positionToMin) {
+ checkFieldRange(positionToMin);
+ return aggregate(ComparableAggregator.getAggregator(positionToMin, getType(),
+ AggregationType.MIN));
+ }
+
+ /**
+ * Applies an aggregation that that gives the current minimum of the pojo
+ * data stream at the given field expression. A field expression is either
+ * the name of a public field or a getter method with parentheses of the
+ * {@link DataStream}S underlying type. A dot can be used to drill down into
+ * objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> min(String field) {
+ return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MIN,
+ false));
+ }
+
+ /**
+ * Applies an aggregation that gives the current maximum of the data stream
+ * at the given position.
+ *
+ * @param positionToMax
+ * The position in the data point to maximize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> max(int positionToMax) {
+ checkFieldRange(positionToMax);
+ return aggregate(ComparableAggregator.getAggregator(positionToMax, getType(),
+ AggregationType.MAX));
+ }
+
+ /**
+ * Applies an aggregation that that gives the current maximum of the pojo
+ * data stream at the given field expression. A field expression is either
+ * the name of a public field or a getter method with parentheses of the
+ * {@link DataStream}S underlying type. A dot can be used to drill down into
+ * objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> max(String field) {
+ return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MAX,
+ false));
+ }
+
+ /**
+ * Applies an aggregation that that gives the current minimum element of the
+ * pojo data stream by the given field expression. A field expression is
+ * either the name of a public field or a getter method with parentheses of
+ * the {@link DataStream}S underlying type. A dot can be used to drill down
+ * into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @param first
+ * If True then in case of field equality the first object will
+ * be returned
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(String field, boolean first) {
+ return aggregate(ComparableAggregator.getAggregator(field, getType(),
+ AggregationType.MINBY, first));
+ }
+
+ /**
+ * Applies an aggregation that that gives the current maximum element of the
+ * pojo data stream by the given field expression. A field expression is
+ * either the name of a public field or a getter method with parentheses of
+ * the {@link DataStream}S underlying type. A dot can be used to drill down
+ * into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @param first
+ * If True then in case of field equality the first object will
+ * be returned
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(String field, boolean first) {
+ return aggregate(ComparableAggregator.getAggregator(field, getType(),
+ AggregationType.MAXBY, first));
+ }
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * minimum value at the given position, if more elements have the minimum
+ * value at the given position, the operator returns the first one by
+ * default.
+ *
+ * @param positionToMinBy
+ * The position in the data point to minimize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
+ return this.minBy(positionToMinBy, true);
+ }
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * minimum value at the given position, if more elements have the minimum
+ * value at the given position, the operator returns the first one by
+ * default.
+ *
+ * @param positionToMinBy
+ * The position in the data point to minimize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(String positionToMinBy) {
+ return this.minBy(positionToMinBy, true);
+ }
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * minimum value at the given position, if more elements have the minimum
+ * value at the given position, the operator returns either the first or
+ * last one, depending on the parameter set.
+ *
+ * @param positionToMinBy
+ * The position in the data point to minimize
+ * @param first
+ * If true, then the operator return the first element with the
+ * minimal value, otherwise returns the last
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy, boolean first) {
+ checkFieldRange(positionToMinBy);
+ return aggregate(ComparableAggregator.getAggregator(positionToMinBy, getType(),
+ AggregationType.MINBY, first));
+ }
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * maximum value at the given position, if more elements have the maximum
+ * value at the given position, the operator returns the first one by
+ * default.
+ *
+ * @param positionToMaxBy
+ * The position in the data point to maximize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
+ return this.maxBy(positionToMaxBy, true);
+ }
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * maximum value at the given position, if more elements have the maximum
+ * value at the given position, the operator returns the first one by
+ * default.
+ *
+ * @param positionToMaxBy
+ * The position in the data point to maximize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(String positionToMaxBy) {
+ return this.maxBy(positionToMaxBy, true);
+ }
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * maximum value at the given position, if more elements have the maximum
+ * value at the given position, the operator returns either the first or
+ * last one, depending on the parameter set.
+ *
+ * @param positionToMaxBy
+ * The position in the data point to maximize.
+ * @param first
+ * If true, then the operator return the first element with the
+ * maximum value, otherwise returns the last
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
+ checkFieldRange(positionToMaxBy);
+ return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getType(),
+ AggregationType.MAXBY, first));
+ }
+
+ /**
+ * Creates a new DataStream containing the current number (count) of
+ * received records.
+ *
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<Long, ?> count() {
+ TypeInformation<Long> outTypeInfo = TypeExtractor.getForObject(Long.valueOf(0));
+
+ return transform("Count", outTypeInfo, new CounterInvokable<OUT>());
+ }
+
+ /**
+ * Create a {@link WindowedDataStream} that can be used to apply
+ * transformation like {@link WindowedDataStream#reduce} or aggregations on
+ * preset chunks(windows) of the data stream. To define the windows one or
+ * more {@link WindowingHelper} such as {@link Time}, {@link Count} and
+ * {@link Delta} can be used.</br></br> When applied to a grouped data
+ * stream, the windows (evictions) and slide sizes (triggers) will be
+ * computed on a per group basis. </br></br> For more advanced control over
+ * the trigger and eviction policies please refer to
+ * {@link #window(triggers, evicters)} </br> </br> For example to create a
+ * sum every 5 seconds in a tumbling fashion:</br>
+ * {@code ds.window(Time.of(5, TimeUnit.SECONDS)).sum(field)} </br></br> To
+ * create sliding windows use the
+ * {@link WindowedDataStream#every(WindowingHelper...)} </br></br> The same
+ * example with 3 second slides:</br>
+ *
+ * {@code ds.window(Time.of(5, TimeUnit.SECONDS)).every(Time.of(3,
+ * TimeUnit.SECONDS)).sum(field)}
+ *
+ * @param policyHelpers
+ * Any {@link WindowingHelper} such as {@link Time},
+ * {@link Count} and {@link Delta} to define the window.
+ * @return A {@link WindowedDataStream} providing further operations.
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public WindowedDataStream<OUT> window(WindowingHelper... policyHelpers) {
+ return new WindowedDataStream<OUT>(this, policyHelpers);
+ }
+
+ /**
+ * Create a {@link WindowedDataStream} using the given {@link TriggerPolicy}
+ * s and {@link EvictionPolicy}s. Windowing can be used to apply
+ * transformation like {@link WindowedDataStream#reduce} or aggregations on
+ * preset chunks(windows) of the data stream.</br></br>For most common
+ * use-cases please refer to {@link #window(WindowingHelper...)}
+ *
+ * @param triggers
+ * The list of {@link TriggerPolicy}s that will determine how
+ * often the user function is called on the window.
+ * @param evicters
+ * The list of {@link EvictionPolicy}s that will determine the
+ * number of elements in each time window.
+ * @return A {@link WindowedDataStream} providing further operations.
+ */
+ public WindowedDataStream<OUT> window(List<TriggerPolicy<OUT>> triggers,
+ List<EvictionPolicy<OUT>> evicters) {
+ return new WindowedDataStream<OUT>(this, triggers, evicters);
+ }
+
+ /**
+ * Writes a DataStream to the standard output stream (stdout).<br>
+ * For each element of the DataStream the result of
+ * {@link Object#toString()} is written.
+ *
+ * @return The closed DataStream.
+ */
+ public DataStreamSink<OUT> print() {
+ PrintSinkFunction<OUT> printFunction = new PrintSinkFunction<OUT>();
+ DataStreamSink<OUT> returnStream = addSink(printFunction);
+
+ return returnStream;
+ }
+
+ /**
+ * Writes a DataStream to the standard output stream (stderr).<br>
+ * For each element of the DataStream the result of
+ * {@link Object#toString()} is written.
+ *
+ * @return The closed DataStream.
+ */
+ public DataStreamSink<OUT> printToErr() {
+ PrintSinkFunction<OUT> printFunction = new PrintSinkFunction<OUT>(true);
+ DataStreamSink<OUT> returnStream = addSink(printFunction);
+
+ return returnStream;
+ }
+
+ /**
+ * Writes a DataStream to the file specified by path in text format. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ *
+ * @return the closed DataStream.
+ */
+ public DataStreamSink<OUT> writeAsText(String path) {
+ return writeToFile(new TextOutputFormat<OUT>(new Path(path)), 0L);
+ }
+
+ /**
+ * Writes a DataStream to the file specified by path in text format. The
+ * writing is performed periodically, in every millis milliseconds. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ * @param millis
+ * the file update frequency
+ *
+ * @return the closed DataStream
+ */
+ public DataStreamSink<OUT> writeAsText(String path, long millis) {
+ TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
+ return writeToFile(tof, millis);
+ }
+
+ /**
+ * Writes a DataStream to the file specified by path in text format. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ * @param writeMode
+ * Control the behavior for existing files. Options are
+ * NO_OVERWRITE and OVERWRITE.
+ *
+ * @return the closed DataStream.
+ */
+ public DataStreamSink<OUT> writeAsText(String path, WriteMode writeMode) {
+ TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
+ tof.setWriteMode(writeMode);
+ return writeToFile(tof, 0L);
+ }
+
+ /**
+ * Writes a DataStream to the file specified by path in text format. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ * @param writeMode
+ * Controls the behavior for existing files. Options are
+ * NO_OVERWRITE and OVERWRITE.
+ * @param millis
+ * the file update frequency
+ *
+ * @return the closed DataStream.
+ */
+ public DataStreamSink<OUT> writeAsText(String path, WriteMode writeMode, long millis) {
+ TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
+ tof.setWriteMode(writeMode);
+ return writeToFile(tof, millis);
+ }
+
+ /**
+ * Writes a DataStream to the file specified by path in csv format. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written. This method can only be used on data streams of tuples.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ *
+ * @return the closed DataStream
+ */
+ @SuppressWarnings("unchecked")
+ public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path) {
+ Validate.isTrue(getType().isTupleType(),
+ "The writeAsCsv() method can only be used on data sets of tuples.");
+ CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+ CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+ return writeToFile((OutputFormat<OUT>) of, 0L);
+ }
+
+ /**
+ * Writes a DataStream to the file specified by path in csv format. The
+ * writing is performed periodically, in every millis milliseconds. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written. This method can only be used on data streams of tuples.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ * @param millis
+ * the file update frequency
+ *
+ * @return the closed DataStream
+ */
+ @SuppressWarnings("unchecked")
+ public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, long millis) {
+ Validate.isTrue(getType().isTupleType(),
+ "The writeAsCsv() method can only be used on data sets of tuples.");
+ CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+ CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+ return writeToFile((OutputFormat<OUT>) of, millis);
+ }
+
+ /**
+ * Writes a DataStream to the file specified by path in csv format. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written. This method can only be used on data streams of tuples.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ * @param writeMode
+ * Controls the behavior for existing files. Options are
+ * NO_OVERWRITE and OVERWRITE.
+ *
+ * @return the closed DataStream
+ */
+ @SuppressWarnings("unchecked")
+ public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, WriteMode writeMode) {
+ Validate.isTrue(getType().isTupleType(),
+ "The writeAsCsv() method can only be used on data sets of tuples.");
+ CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+ CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+ if (writeMode != null) {
+ of.setWriteMode(writeMode);
+ }
+ return writeToFile((OutputFormat<OUT>) of, 0L);
+ }
+
+ /**
+ * Writes a DataStream to the file specified by path in csv format. The
+ * writing is performed periodically, in every millis milliseconds. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written. This method can only be used on data streams of tuples.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ * @param writeMode
+ * Controls the behavior for existing files. Options are
+ * NO_OVERWRITE and OVERWRITE.
+ * @param millis
+ * the file update frequency
+ *
+ * @return the closed DataStream
+ */
+ @SuppressWarnings("unchecked")
+ public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, WriteMode writeMode,
+ long millis) {
+ Validate.isTrue(getType().isTupleType(),
+ "The writeAsCsv() method can only be used on data sets of tuples.");
+ CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+ CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+ if (writeMode != null) {
+ of.setWriteMode(writeMode);
+ }
+ return writeToFile((OutputFormat<OUT>) of, millis);
+ }
+
+ private DataStreamSink<OUT> writeToFile(OutputFormat<OUT> format, long millis) {
+ DataStreamSink<OUT> returnStream = addSink(new FileSinkFunctionByMillis<OUT>(format, millis));
+ return returnStream;
+ }
+
+ protected SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregate) {
+
+ StreamReduceInvokable<OUT> invokable = new StreamReduceInvokable<OUT>(aggregate);
+
+ SingleOutputStreamOperator<OUT, ?> returnStream = transform("Aggregation", getType(),
+ invokable);
+
+ return returnStream;
+ }
+
+ /**
+ * Method for passing user defined invokables along with the type
+ * informations that will transform the DataStream.
+ *
+ * @param operatorName
+ * name of the operator, for logging purposes
+ * @param outTypeInfo
+ * the output type of the operator
+ * @param invokable
+ * the object containing the transformation logic
+ * @param <R>
+ * type of the return stream
+ * @return the data stream constructed
+ */
+ public <R> SingleOutputStreamOperator<R, ?> transform(String operatorName,
+ TypeInformation<R> outTypeInfo, StreamInvokable<OUT, R> invokable) {
+ DataStream<OUT> inputStream = this.copy();
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ SingleOutputStreamOperator<R, ?> returnStream = new SingleOutputStreamOperator(environment,
+ operatorName, outTypeInfo, invokable);
+
+ streamGraph.addStreamVertex(returnStream.getId(), invokable, getType(), outTypeInfo,
+ operatorName, returnStream.getParallelism());
+
+ connectGraph(inputStream, returnStream.getId(), 0);
+
+ return returnStream;
+ }
+
+ /**
+ * Internal function for setting the partitioner for the DataStream
+ *
+ * @param partitioner
+ * Partitioner to set.
+ * @return The modified DataStream.
+ */
+ protected DataStream<OUT> setConnectionType(StreamPartitioner<OUT> partitioner) {
+ DataStream<OUT> returnStream = this.copy();
+
+ for (DataStream<OUT> stream : returnStream.mergedStreams) {
+ stream.partitioner = partitioner;
+ }
+
+ return returnStream;
+ }
+
+ /**
+ * Internal function for assembling the underlying
+ * {@link org.apache.flink.runtime.jobgraph.JobGraph} of the job. Connects
+ * the outputs of the given input stream to the specified output stream
+ * given by the outputID.
+ *
+ * @param inputStream
+ * input data stream
+ * @param outputID
+ * ID of the output
+ * @param typeNumber
+ * Number of the type (used at co-functions)
+ */
+ protected <X> void connectGraph(DataStream<X> inputStream, String outputID, int typeNumber) {
+ for (DataStream<X> stream : inputStream.mergedStreams) {
+ streamGraph.setEdge(stream.getId(), outputID, stream.partitioner, typeNumber,
+ inputStream.userDefinedNames);
+ }
+
+ }
+
+ /**
+ * Adds the given sink to this DataStream. Only streams with sinks added
+ * will be executed once the {@link StreamExecutionEnvironment#execute()}
+ * method is called.
+ *
+ * @param sinkFunction
+ * The object containing the sink's invoke function.
+ * @return The closed DataStream.
+ */
+ public DataStreamSink<OUT> addSink(SinkFunction<OUT> sinkFunction) {
+
+ StreamInvokable<OUT, OUT> sinkInvokable = new SinkInvokable<OUT>(clean(sinkFunction));
+
+ DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", getType(),
+ sinkInvokable);
+
+ streamGraph.addStreamVertex(returnStream.getId(), sinkInvokable, getType(), null,
+ "Stream Sink", returnStream.getParallelism());
+
+ this.connectGraph(this.copy(), returnStream.getId(), 0);
+
+ return returnStream;
+ }
+
+ /**
+ * Gets the class of the field at the given position
+ *
+ * @param pos
+ * Position of the field
+ * @return The class of the field
+ */
+ @SuppressWarnings("rawtypes")
+ protected Class<?> getClassAtPos(int pos) {
+ Class<?> type;
+ TypeInformation<OUT> outTypeInfo = getType();
+ if (outTypeInfo.isTupleType()) {
+ type = ((TupleTypeInfo) outTypeInfo).getTypeAt(pos).getTypeClass();
+
+ } else if (outTypeInfo instanceof BasicArrayTypeInfo) {
+
+ type = ((BasicArrayTypeInfo) outTypeInfo).getComponentTypeClass();
+
+ } else if (outTypeInfo instanceof PrimitiveArrayTypeInfo) {
+ Class<?> clazz = outTypeInfo.getTypeClass();
+ if (clazz == boolean[].class) {
+ type = Boolean.class;
+ } else if (clazz == short[].class) {
+ type = Short.class;
+ } else if (clazz == int[].class) {
+ type = Integer.class;
+ } else if (clazz == long[].class) {
+ type = Long.class;
+ } else if (clazz == float[].class) {
+ type = Float.class;
+ } else if (clazz == double[].class) {
+ type = Double.class;
+ } else if (clazz == char[].class) {
+ type = Character.class;
+ } else {
+ throw new IndexOutOfBoundsException("Type could not be determined for array");
+ }
+
+ } else if (pos == 0) {
+ type = outTypeInfo.getTypeClass();
+ } else {
+ throw new IndexOutOfBoundsException("Position is out of range");
+ }
+ return type;
+ }
+
+ /**
+ * Checks if the given field position is allowed for the output type
+ *
+ * @param pos
+ * Position to check
+ */
+ protected void checkFieldRange(int pos) {
+ try {
+ getClassAtPos(pos);
+ } catch (IndexOutOfBoundsException e) {
+ throw new RuntimeException("Selected field is out of range");
+
+ }
+ }
+
+ private void validateMerge(String id) {
+ for (DataStream<OUT> ds : this.mergedStreams) {
+ if (ds.getId().equals(id)) {
+ throw new RuntimeException("A DataStream cannot be merged with itself");
+ }
+ }
+ }
+
+ /**
+ * Creates a copy of the {@link DataStream}
+ *
+ * @return The copy
+ */
+ public DataStream<OUT> copy() {
+ return new DataStream<OUT>(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
new file mode 100755
index 0000000..f064332
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+
+/**
+ * Represents the end of a DataStream.
+ *
+ * @param <IN>
+ * The type of the DataStream closed by the sink.
+ */
+public class DataStreamSink<IN> extends SingleOutputStreamOperator<IN, DataStreamSink<IN>> {
+
+ protected DataStreamSink(StreamExecutionEnvironment environment, String operatorType,
+ TypeInformation<IN> outTypeInfo, StreamInvokable<?,?> invokable) {
+ super(environment, operatorType, outTypeInfo, invokable);
+ }
+
+ protected DataStreamSink(DataStream<IN> dataStream) {
+ super(dataStream);
+ }
+
+ @Override
+ public DataStreamSink<IN> copy() {
+ throw new RuntimeException("Data stream sinks cannot be copied");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
new file mode 100755
index 0000000..b596cbd
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+
+/**
+ * The DataStreamSource represents the starting point of a DataStream.
+ *
+ * @param <OUT>
+ * Type of the DataStream created.
+ */
+public class DataStreamSource<OUT> extends SingleOutputStreamOperator<OUT, DataStreamSource<OUT>> {
+
+ boolean isParallel;
+
+ public DataStreamSource(StreamExecutionEnvironment environment, String operatorType,
+ TypeInformation<OUT> outTypeInfo, StreamInvokable<?, ?> invokable, boolean isParallel) {
+ super(environment, operatorType, outTypeInfo, invokable);
+ this.isParallel = isParallel;
+ }
+
+ @Override
+ public DataStreamSource<OUT> setParallelism(int dop) {
+ if (dop > 1 && !isParallel) {
+ throw new IllegalArgumentException("Source: " + this.id + " is not a parallel source");
+ } else {
+ return (DataStreamSource<OUT>) super.setParallelism(dop);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
new file mode 100755
index 0000000..c871c20
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.RichReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
+import org.apache.flink.streaming.api.invokable.operator.GroupedReduceInvokable;
+import org.apache.flink.streaming.partitioner.StreamPartitioner;
+
+/**
+ * A GroupedDataStream represents a {@link DataStream} which has been
+ * partitioned by the given {@link KeySelector}. Operators like {@link #reduce},
+ * {@link #batchReduce} etc. can be applied on the {@link GroupedDataStream} to
+ * get additional functionality by the grouping.
+ *
+ * @param <OUT>
+ * The output type of the {@link GroupedDataStream}.
+ */
+public class GroupedDataStream<OUT> extends DataStream<OUT> {
+
+ KeySelector<OUT, ?> keySelector;
+
+ protected GroupedDataStream(DataStream<OUT> dataStream, KeySelector<OUT, ?> keySelector) {
+ super(dataStream.partitionBy(keySelector));
+ this.keySelector = keySelector;
+ }
+
+ protected GroupedDataStream(GroupedDataStream<OUT> dataStream) {
+ super(dataStream);
+ this.keySelector = dataStream.keySelector;
+ }
+
+ public KeySelector<OUT, ?> getKeySelector() {
+ return this.keySelector;
+ }
+
+ /**
+ * Applies a reduce transformation on the grouped data stream grouped on by
+ * the given key position. The {@link ReduceFunction} will receive input
+ * values based on the key value. Only input values with the same key will
+ * go to the same reducer.The user can also extend
+ * {@link RichReduceFunction} to gain access to other features provided by
+ * the {@link RichFuntion} interface.
+ *
+ * @param reducer
+ * The {@link ReduceFunction} that will be called for every
+ * element of the input values with the same key.
+ * @return The transformed DataStream.
+ */
+ @Override
+ public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reducer) {
+ return transform("Grouped Reduce", getType(), new GroupedReduceInvokable<OUT>(clean(reducer),
+ keySelector));
+ }
+
+ /**
+ * Applies an aggregation that sums the grouped data stream at the given
+ * position, grouped by the given key position. Input values with the same
+ * key will be summed.
+ *
+ * @param positionToSum
+ * The position in the data point to sum
+ * @return The transformed DataStream.
+ */
+ @Override
+ public SingleOutputStreamOperator<OUT, ?> sum(final int positionToSum) {
+ return super.sum(positionToSum);
+ }
+
+ /**
+ * Applies an aggregation that gives the minimum of the grouped data stream
+ * at the given position, grouped by the given key position. Input values
+ * with the same key will be minimized.
+ *
+ * @param positionToMin
+ * The position in the data point to minimize
+ * @return The transformed DataStream.
+ */
+ @Override
+ public SingleOutputStreamOperator<OUT, ?> min(final int positionToMin) {
+ return super.min(positionToMin);
+ }
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * minimum value at the given position for each group on a grouped data
+ * stream. If more elements have the minimum value at the given position,
+ * the operator returns the first one by default.
+ *
+ * @param positionToMinBy
+ * The position in the data point to minimize
+ * @return The transformed DataStream.
+ */
+ @Override
+ public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
+ return super.minBy(positionToMinBy);
+ }
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * minimum value at the given position for each group on a grouped data
+ * stream. If more elements have the minimum value at the given position,
+ * the operator returns either the first or last one depending on the
+ * parameters.
+ *
+ * @param positionToMinBy
+ * The position in the data point to minimize
+ * @param first
+ * If true, then the operator return the first element with the
+ * maximum value, otherwise returns the last
+ * @return The transformed DataStream.
+ */
+ @Override
+ public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy, boolean first) {
+ return super.minBy(positionToMinBy, first);
+ }
+
+ /**
+ * Applies an aggregation that gives the maximum of the grouped data stream
+ * at the given position, grouped by the given key position. Input values
+ * with the same key will be maximized.
+ *
+ * @param positionToMax
+ * The position in the data point to maximize
+ * @return The transformed DataStream.
+ */
+ @Override
+ public SingleOutputStreamOperator<OUT, ?> max(final int positionToMax) {
+ return super.max(positionToMax);
+ }
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * maximum value at the given position for each group on a grouped data
+ * stream. If more elements have the maximum value at the given position,
+ * the operator returns the first one by default.
+ *
+ * @param positionToMaxBy
+ * The position in the data point to maximize
+ * @return The transformed DataStream.
+ */
+ @Override
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
+ return super.maxBy(positionToMaxBy);
+ }
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * maximum value at the given position for each group on a grouped data
+ * stream. If more elements have the maximum value at the given position,
+ * the operator returns either the first or last one depending on the
+ * parameters.
+ *
+ * @param positionToMaxBy
+ * The position in the data point to maximize
+ * @param first
+ * If true, then the operator return the first element with the
+ * maximum value, otherwise returns the last
+ * @return The transformed DataStream.
+ */
+ @Override
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
+ return super.maxBy(positionToMaxBy, first);
+ }
+
+ @Override
+ protected SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregate) {
+
+ GroupedReduceInvokable<OUT> invokable = new GroupedReduceInvokable<OUT>(clean(aggregate),
+ keySelector);
+
+ SingleOutputStreamOperator<OUT, ?> returnStream = transform("Grouped Aggregation", getType(),
+ invokable);
+
+ return returnStream;
+ }
+
+ @Override
+ protected DataStream<OUT> setConnectionType(StreamPartitioner<OUT> partitioner) {
+ System.out.println("Setting the partitioning after groupBy can affect the grouping");
+ return super.setConnectionType(partitioner);
+ }
+
+ @Override
+ public GroupedDataStream<OUT> copy() {
+ return new GroupedDataStream<OUT>(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
new file mode 100644
index 0000000..306bfe8
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+
+/**
+ * The iterative data stream represents the start of an iteration in a
+ * {@link DataStream}.
+ *
+ * @param <IN>
+ * Type of the DataStream
+ */
+public class IterativeDataStream<IN> extends
+ SingleOutputStreamOperator<IN, IterativeDataStream<IN>> {
+
+ static Integer iterationCount = 0;
+ protected Integer iterationID;
+ protected long waitTime;
+
+ protected IterativeDataStream(DataStream<IN> dataStream, long maxWaitTime) {
+ super(dataStream);
+ setBufferTimeout(dataStream.environment.getBufferTimeout());
+ iterationID = iterationCount;
+ iterationCount++;
+ waitTime = maxWaitTime;
+ }
+
+ protected IterativeDataStream(DataStream<IN> dataStream, Integer iterationID, long waitTime) {
+ super(dataStream);
+ this.iterationID = iterationID;
+ this.waitTime = waitTime;
+ }
+
+ /**
+ * Closes the iteration. This method defines the end of the iterative
+ * program part that will be fed back to the start of the iteration. </br>
+ * </br>A common usage pattern for streaming iterations is to use output
+ * splitting to send a part of the closing data stream to the head. Refer to
+ * {@link SingleOutputStreamOperator#split(OutputSelector)} for more
+ * information.
+ *
+ *
+ * @param iterationResult
+ * The data stream that is fed back to the next iteration head.
+ * @return Returns the stream that was fed back to the iteration. In most
+ * cases no further transformation are applied on this stream.
+ *
+ */
+ public DataStream<IN> closeWith(DataStream<IN> iterationTail) {
+ DataStream<IN> iterationSink = new DataStreamSink<IN>(environment, "Iteration Sink", null,
+ null);
+
+ // We add an iteration sink to the tail which will send tuples to the
+ // iteration head
+ streamGraph.addIterationTail(iterationSink.getId(), iterationTail.getId(), iterationID,
+ waitTime);
+
+ connectGraph(iterationTail.forward(), iterationSink.getId(), 0);
+ return iterationTail;
+ }
+
+ @Override
+ public <R> SingleOutputStreamOperator<R, ?> transform(String operatorName,
+ TypeInformation<R> outTypeInfo, StreamInvokable<IN, R> invokable) {
+
+ // We call the superclass tranform method
+ SingleOutputStreamOperator<R, ?> returnStream = super.transform(operatorName, outTypeInfo,
+ invokable);
+
+ // Then we add a source that will take care of receiving feedback tuples
+ // from the tail
+ addIterationSource(returnStream);
+
+ return returnStream;
+ }
+
+ private <X> void addIterationSource(DataStream<X> dataStream) {
+
+ DataStream<X> iterationSource = new DataStreamSource<X>(environment, "Iteration Source",
+ null, null, true);
+
+ streamGraph.addIterationHead(iterationSource.getId(), dataStream.getId(), iterationID,
+ dataStream.getParallelism(), waitTime);
+ }
+
+ @Override
+ public IterativeDataStream<IN> copy() {
+ return new IterativeDataStream<IN>(this, iterationID, waitTime);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
new file mode 100755
index 0000000..dcfd6fe
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
+import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
+import org.apache.flink.streaming.state.OperatorState;
+
+/**
+ * The SingleOutputStreamOperator represents a user defined transformation
+ * applied on a {@link DataStream} with one predefined output type.
+ *
+ * @param <OUT>
+ * Output type of the operator.
+ * @param <O>
+ * Type of the operator.
+ */
+public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperator<OUT, O>> extends
+ DataStream<OUT> {
+
+ protected boolean isSplit;
+ protected StreamInvokable<?, ?> invokable;
+
+ protected SingleOutputStreamOperator(StreamExecutionEnvironment environment,
+ String operatorType, TypeInformation<OUT> outTypeInfo, StreamInvokable<?, ?> invokable) {
+ super(environment, operatorType, outTypeInfo);
+ setBufferTimeout(environment.getBufferTimeout());
+ this.isSplit = false;
+ this.invokable = invokable;
+ }
+
+ @SuppressWarnings("unchecked")
+ protected SingleOutputStreamOperator(DataStream<OUT> dataStream) {
+ super(dataStream);
+ if (dataStream instanceof SingleOutputStreamOperator) {
+ this.isSplit = ((SingleOutputStreamOperator<OUT, ?>) dataStream).isSplit;
+ this.invokable = ((SingleOutputStreamOperator<OUT, ?>) dataStream).invokable;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public <R> SingleOutputStreamOperator<R, ?> setType(TypeInformation<R> outType) {
+ streamGraph.setOutType(id, outType);
+ typeInfo = outType;
+ return (SingleOutputStreamOperator<R, ?>) this;
+ }
+
+ /**
+ * Sets the degree of parallelism for this operator. The degree must be 1 or
+ * more.
+ *
+ * @param dop
+ * The degree of parallelism for this operator.
+ * @return The operator with set degree of parallelism.
+ */
+ public SingleOutputStreamOperator<OUT, O> setParallelism(int dop) {
+ if (dop < 1) {
+ throw new IllegalArgumentException("The parallelism of an operator must be at least 1.");
+ }
+ this.degreeOfParallelism = dop;
+
+ streamGraph.setParallelism(id, degreeOfParallelism);
+
+ return this;
+ }
+
+ /**
+ * Sets the maximum time frequency (ms) for the flushing of the output
+ * buffer. By default the output buffers flush only when they are full.
+ *
+ * @param timeoutMillis
+ * The maximum time between two output flushes.
+ * @return The operator with buffer timeout set.
+ */
+ public SingleOutputStreamOperator<OUT, O> setBufferTimeout(long timeoutMillis) {
+ streamGraph.setBufferTimeout(id, timeoutMillis);
+ return this;
+ }
+
+ /**
+ * This is a beta feature </br></br> Register an operator state for this
+ * operator by the given name. This name can be used to retrieve the state
+ * during runtime using {@link StreamingRuntimeContext#getState(String)}. To
+ * obtain the {@link StreamingRuntimeContext} from the user-defined function
+ * use the {@link RichFunction#getRuntimeContext()} method.
+ *
+ * @param name
+ * The name of the operator state.
+ * @param state
+ * The state to be registered for this name.
+ * @return The data stream with state registered.
+ */
+ protected SingleOutputStreamOperator<OUT, O> registerState(String name, OperatorState<?> state) {
+ streamGraph.addOperatorState(getId(), name, state);
+ return this;
+ }
+
+ /**
+ * This is a beta feature </br></br> Register operator states for this
+ * operator provided in a map. The registered states can be retrieved during
+ * runtime using {@link StreamingRuntimeContext#getState(String)}. To obtain
+ * the {@link StreamingRuntimeContext} from the user-defined function use
+ * the {@link RichFunction#getRuntimeContext()} method.
+ *
+ * @param states
+ * The map containing the states that will be registered.
+ * @return The data stream with states registered.
+ */
+ protected SingleOutputStreamOperator<OUT, O> registerState(Map<String, OperatorState<?>> states) {
+ for (Entry<String, OperatorState<?>> entry : states.entrySet()) {
+ streamGraph.addOperatorState(getId(), entry.getKey(), entry.getValue());
+ }
+
+ return this;
+ }
+
+ @SuppressWarnings("unchecked")
+ public SingleOutputStreamOperator<OUT, O> broadcast() {
+ return (SingleOutputStreamOperator<OUT, O>) super.broadcast();
+ }
+
+ @SuppressWarnings("unchecked")
+ public SingleOutputStreamOperator<OUT, O> shuffle() {
+ return (SingleOutputStreamOperator<OUT, O>) super.shuffle();
+ }
+
+ @SuppressWarnings("unchecked")
+ public SingleOutputStreamOperator<OUT, O> forward() {
+ return (SingleOutputStreamOperator<OUT, O>) super.forward();
+ }
+
+ @SuppressWarnings("unchecked")
+ public SingleOutputStreamOperator<OUT, O> distribute() {
+ return (SingleOutputStreamOperator<OUT, O>) super.distribute();
+ }
+
+ @SuppressWarnings("unchecked")
+ public SingleOutputStreamOperator<OUT, O> global() {
+ return (SingleOutputStreamOperator<OUT, O>) super.global();
+ }
+
+ @Override
+ public SingleOutputStreamOperator<OUT, O> copy() {
+ return new SingleOutputStreamOperator<OUT, O>(this);
+ }
+
+ public SingleOutputStreamOperator<OUT, O> setChainingStrategy(ChainingStrategy strategy) {
+ this.invokable.setChainingStrategy(strategy);
+ return this;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
new file mode 100755
index 0000000..97458a8
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import java.util.Arrays;
+
+import org.apache.flink.streaming.api.collector.OutputSelector;
+
+/**
+ * The SplitDataStream represents an operator that has been split using an
+ * {@link OutputSelector}. Named outputs can be selected using the
+ * {@link #select} function. To apply transformation on the whole output simply
+ * call the transformation on the SplitDataStream
+ *
+ * @param <OUT>
+ * The type of the output.
+ */
+public class SplitDataStream<OUT> extends DataStream<OUT> {
+
+ protected SplitDataStream(DataStream<OUT> dataStream) {
+ super(dataStream);
+ }
+
+ /**
+ * Sets the output names for which the next operator will receive values.
+ *
+ * @param outputNames
+ * The output names for which the operator will receive the
+ * input.
+ * @return Returns the selected DataStream
+ */
+ public DataStream<OUT> select(String... outputNames) {
+ return selectOutput(outputNames);
+ }
+
+ private DataStream<OUT> selectOutput(String[] outputNames) {
+ for (String outName : outputNames) {
+ if (outName == null) {
+ throw new RuntimeException("Selected names must not be null");
+ }
+ }
+
+ DataStream<OUT> returnStream = copy();
+
+ for (DataStream<OUT> ds : returnStream.mergedStreams) {
+ ds.userDefinedNames = Arrays.asList(outputNames);
+ }
+ return returnStream;
+ }
+
+}
[11/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
new file mode 100644
index 0000000..ed246c8
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+public class WindowReduceInvokable<IN> extends WindowInvokable<IN, IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ ReduceFunction<IN> reducer;
+
+ public WindowReduceInvokable(ReduceFunction<IN> userFunction,
+ LinkedList<TriggerPolicy<IN>> triggerPolicies,
+ LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+ super(userFunction, triggerPolicies, evictionPolicies);
+ this.reducer = userFunction;
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ Iterator<IN> reducedIterator = buffer.iterator();
+ IN reduced = null;
+
+ while (reducedIterator.hasNext() && reduced == null) {
+ reduced = reducedIterator.next();
+ }
+
+ while (reducedIterator.hasNext()) {
+ IN next = reducedIterator.next();
+ if (next != null) {
+ reduced = reducer.reduce(copy(reduced), copy(next));
+ }
+ }
+ if (reduced != null) {
+ collector.collect(reduced);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapInvokable.java
new file mode 100644
index 0000000..4cbaebb
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapInvokable.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
+
+public class CoFlatMapInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ private CoFlatMapFunction<IN1, IN2, OUT> flatMapper;
+
+ public CoFlatMapInvokable(CoFlatMapFunction<IN1, IN2, OUT> flatMapper) {
+ super(flatMapper);
+ this.flatMapper = flatMapper;
+ }
+
+ @Override
+ public void handleStream1() throws Exception {
+ callUserFunctionAndLogException1();
+ }
+
+ @Override
+ public void handleStream2() throws Exception {
+ callUserFunctionAndLogException2();
+ }
+
+ @Override
+ protected void callUserFunction1() throws Exception {
+ flatMapper.flatMap1(reuse1.getObject(), collector);
+
+ }
+
+ @Override
+ protected void callUserFunction2() throws Exception {
+ flatMapper.flatMap2(reuse2.getObject(), collector);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceInvokable.java
new file mode 100644
index 0000000..4907ac5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceInvokable.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.function.co.CoReduceFunction;
+
+public class CoGroupedReduceInvokable<IN1, IN2, OUT> extends CoReduceInvokable<IN1, IN2, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ protected KeySelector<IN1, ?> keySelector1;
+ protected KeySelector<IN2, ?> keySelector2;
+ private Map<Object, IN1> values1;
+ private Map<Object, IN2> values2;
+ IN1 reduced1;
+ IN2 reduced2;
+
+ public CoGroupedReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer,
+ KeySelector<IN1, ?> keySelector1, KeySelector<IN2, ?> keySelector2) {
+ super(coReducer);
+ this.coReducer = coReducer;
+ this.keySelector1 = keySelector1;
+ this.keySelector2 = keySelector2;
+ values1 = new HashMap<Object, IN1>();
+ values2 = new HashMap<Object, IN2>();
+ }
+
+ @Override
+ public void handleStream1() throws Exception {
+ Object key = reuse1.getKey(keySelector1);
+ currentValue1 = values1.get(key);
+ nextValue1 = reuse1.getObject();
+ if (currentValue1 != null) {
+ callUserFunctionAndLogException1();
+ values1.put(key, reduced1);
+ collector.collect(coReducer.map1(reduced1));
+ } else {
+ values1.put(key, nextValue1);
+ collector.collect(coReducer.map1(nextValue1));
+ }
+ }
+
+ @Override
+ public void handleStream2() throws Exception {
+ Object key = reuse2.getKey(keySelector2);
+ currentValue2 = values2.get(key);
+ nextValue2 = reuse2.getObject();
+ if (currentValue2 != null) {
+ callUserFunctionAndLogException2();
+ values2.put(key, reduced2);
+ collector.collect(coReducer.map2(reduced2));
+ } else {
+ values2.put(key, nextValue2);
+ collector.collect(coReducer.map2(nextValue2));
+ }
+ }
+
+ @Override
+ protected void callUserFunction1() throws Exception {
+ reduced1 = coReducer.reduce1(currentValue1, nextValue1);
+
+ }
+
+ @Override
+ protected void callUserFunction2() throws Exception {
+ reduced2 = coReducer.reduce2(currentValue2, nextValue2);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java
new file mode 100644
index 0000000..604873e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.api.streamvertex.StreamTaskContext;
+import org.apache.flink.streaming.io.CoReaderIterator;
+import org.apache.flink.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class CoInvokable<IN1, IN2, OUT> extends StreamInvokable<IN1, OUT> {
+
+ public CoInvokable(Function userFunction) {
+ super(userFunction);
+ }
+
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(CoInvokable.class);
+
+ protected CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>> recordIterator;
+ protected StreamRecord<IN1> reuse1;
+ protected StreamRecord<IN2> reuse2;
+ protected StreamRecordSerializer<IN1> srSerializer1;
+ protected StreamRecordSerializer<IN2> srSerializer2;
+ protected TypeSerializer<IN1> serializer1;
+ protected TypeSerializer<IN2> serializer2;
+
+ @Override
+ public void setup(StreamTaskContext<OUT> taskContext) {
+ this.collector = taskContext.getOutputCollector();
+
+ this.recordIterator = taskContext.getCoReader();
+
+ this.srSerializer1 = taskContext.getInputSerializer(0);
+ this.srSerializer2 = taskContext.getInputSerializer(1);
+
+ this.reuse1 = srSerializer1.createInstance();
+ this.reuse2 = srSerializer2.createInstance();
+
+ this.serializer1 = srSerializer1.getObjectSerializer();
+ this.serializer2 = srSerializer2.getObjectSerializer();
+ }
+
+ protected void resetReuseAll() {
+ this.reuse1 = srSerializer1.createInstance();
+ this.reuse2 = srSerializer2.createInstance();
+ }
+
+ protected void resetReuse1() {
+ this.reuse1 = srSerializer1.createInstance();
+ }
+
+ protected void resetReuse2() {
+ this.reuse2 = srSerializer2.createInstance();
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ while (true) {
+ int next = recordIterator.next(reuse1, reuse2);
+ if (next == 0) {
+ break;
+ } else if (next == 1) {
+ initialize1();
+ handleStream1();
+ resetReuse1();
+ } else {
+ initialize2();
+ handleStream2();
+ resetReuse2();
+ }
+ }
+ }
+
+ protected abstract void handleStream1() throws Exception;
+
+ protected abstract void handleStream2() throws Exception;
+
+ protected abstract void callUserFunction1() throws Exception;
+
+ protected abstract void callUserFunction2() throws Exception;
+
+ protected void initialize1() {
+
+ };
+
+ protected void initialize2() {
+
+ };
+
+ protected void callUserFunctionAndLogException1() {
+ try {
+ callUserFunction1();
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Calling user function failed due to: {}",
+ StringUtils.stringifyException(e));
+ }
+ }
+ }
+
+ protected void callUserFunctionAndLogException2() {
+ try {
+ callUserFunction2();
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Calling user function failed due to: {}",
+ StringUtils.stringifyException(e));
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapInvokable.java
new file mode 100644
index 0000000..5499dba
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapInvokable.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import org.apache.flink.streaming.api.function.co.CoMapFunction;
+
+public class CoMapInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ private CoMapFunction<IN1, IN2, OUT> mapper;
+
+ public CoMapInvokable(CoMapFunction<IN1, IN2, OUT> mapper) {
+ super(mapper);
+ this.mapper = mapper;
+ }
+
+ @Override
+ public void handleStream1() throws Exception {
+ callUserFunctionAndLogException1();
+ }
+
+ @Override
+ public void handleStream2() throws Exception {
+ callUserFunctionAndLogException2();
+ }
+
+ @Override
+ protected void callUserFunction1() throws Exception {
+ collector.collect(mapper.map1(reuse1.getObject()));
+
+ }
+
+ @Override
+ protected void callUserFunction2() throws Exception {
+ collector.collect(mapper.map2(reuse2.getObject()));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java
new file mode 100644
index 0000000..057dfce
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import org.apache.flink.streaming.api.function.co.CoReduceFunction;
+
+public class CoReduceInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ protected CoReduceFunction<IN1, IN2, OUT> coReducer;
+ protected IN1 currentValue1 = null;
+ protected IN2 currentValue2 = null;
+ protected IN1 nextValue1 = null;
+ protected IN2 nextValue2 = null;
+
+ public CoReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer) {
+ super(coReducer);
+ this.coReducer = coReducer;
+ currentValue1 = null;
+ currentValue2 = null;
+ }
+
+ @Override
+ public void handleStream1() throws Exception {
+ nextValue1 = reuse1.getObject();
+ callUserFunctionAndLogException1();
+ }
+
+ @Override
+ public void handleStream2() throws Exception {
+ nextValue2 = reuse2.getObject();
+ callUserFunctionAndLogException2();
+ }
+
+ @Override
+ protected void callUserFunction1() throws Exception {
+ if (currentValue1 != null) {
+ currentValue1 = coReducer.reduce1(currentValue1, nextValue1);
+ } else {
+ currentValue1 = nextValue1;
+ }
+ collector.collect(coReducer.map1(currentValue1));
+ }
+
+ @Override
+ protected void callUserFunction2() throws Exception {
+ if (currentValue2 != null) {
+ currentValue2 = coReducer.reduce2(currentValue2, nextValue2);
+ } else {
+ currentValue2 = nextValue2;
+ }
+ collector.collect(coReducer.map2(currentValue2));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
new file mode 100644
index 0000000..93f597f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.math.util.MathUtils;
+import org.apache.flink.streaming.api.function.co.CoWindowFunction;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.apache.flink.streaming.state.CircularFifoList;
+
+public class CoWindowInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ protected CoWindowFunction<IN1, IN2, OUT> coWindowFunction;
+ protected long windowSize;
+ protected long slideSize;
+ protected CircularFifoList<StreamRecord<IN1>> circularList1;
+ protected CircularFifoList<StreamRecord<IN2>> circularList2;
+ protected TimestampWrapper<IN1> timeStamp1;
+ protected TimestampWrapper<IN2> timeStamp2;
+
+ protected StreamWindow window;
+
+ protected long startTime;
+ protected long nextRecordTime;
+
+ public CoWindowInvokable(CoWindowFunction<IN1, IN2, OUT> coWindowFunction, long windowSize,
+ long slideInterval, TimestampWrapper<IN1> timeStamp1, TimestampWrapper<IN2> timeStamp2) {
+ super(coWindowFunction);
+ this.coWindowFunction = coWindowFunction;
+ this.windowSize = windowSize;
+ this.slideSize = slideInterval;
+ this.circularList1 = new CircularFifoList<StreamRecord<IN1>>();
+ this.circularList2 = new CircularFifoList<StreamRecord<IN2>>();
+ this.timeStamp1 = timeStamp1;
+ this.timeStamp2 = timeStamp2;
+ this.startTime = timeStamp1.getStartTime();
+
+ this.window = new StreamWindow();
+ }
+
+ @Override
+ protected void handleStream1() throws Exception {
+ window.addToBuffer1(reuse1.getObject());
+ }
+
+ @Override
+ protected void handleStream2() throws Exception {
+ window.addToBuffer2(reuse2.getObject());
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+
+ List<IN1> first = new ArrayList<IN1>();
+ List<IN2> second = new ArrayList<IN2>();
+
+ for (IN1 element : window.circularList1.getElements()) {
+ first.add(serializer1.copy(element));
+ }
+ for (IN2 element : window.circularList2.getElements()) {
+ second.add(serializer2.copy(element));
+ }
+
+ if (!window.circularList1.isEmpty() || !window.circularList2.isEmpty()) {
+ coWindowFunction.coWindow(first, second, collector);
+ }
+ }
+
+ protected class StreamWindow implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ protected int granularity;
+ protected int batchPerSlide;
+ protected long numberOfBatches;
+
+ protected long minibatchCounter;
+
+ protected CircularFifoList<IN1> circularList1;
+ protected CircularFifoList<IN2> circularList2;
+
+ public StreamWindow() {
+ this.granularity = (int) MathUtils.gcd(windowSize, slideSize);
+ this.batchPerSlide = (int) (slideSize / granularity);
+ this.numberOfBatches = windowSize / granularity;
+ this.circularList1 = new CircularFifoList<IN1>();
+ this.circularList2 = new CircularFifoList<IN2>();
+ this.minibatchCounter = 0;
+ }
+
+ public void addToBuffer1(IN1 nextValue) throws Exception {
+ checkWindowEnd(timeStamp1.getTimestamp(nextValue));
+ if (minibatchCounter >= 0) {
+ circularList1.add(nextValue);
+ }
+ }
+
+ public void addToBuffer2(IN2 nextValue) throws Exception {
+ checkWindowEnd(timeStamp2.getTimestamp(nextValue));
+ if (minibatchCounter >= 0) {
+ circularList2.add(nextValue);
+ }
+ }
+
+ protected synchronized void checkWindowEnd(long timeStamp) {
+ nextRecordTime = timeStamp;
+
+ while (miniBatchEnd()) {
+ circularList1.newSlide();
+ circularList2.newSlide();
+ minibatchCounter++;
+ if (windowEnd()) {
+ callUserFunctionAndLogException();
+ circularList1.shiftWindow(batchPerSlide);
+ circularList2.shiftWindow(batchPerSlide);
+ }
+ }
+ }
+
+ protected boolean miniBatchEnd() {
+ if (nextRecordTime < startTime + granularity) {
+ return false;
+ } else {
+ startTime += granularity;
+ return true;
+ }
+ }
+
+ public boolean windowEnd() {
+ if (minibatchCounter == numberOfBatches) {
+ minibatchCounter -= batchPerSlide;
+ return true;
+ }
+ return false;
+ }
+
+ public void reduceLastBatch() {
+ if (!miniBatchEnd()) {
+ callUserFunctionAndLogException();
+ }
+ }
+
+ public Iterable<IN1> getIterable1() {
+ return circularList1.getIterable();
+ }
+
+ public Iterable<IN2> getIterable2() {
+ return circularList2.getIterable();
+ }
+
+ @Override
+ public String toString() {
+ return circularList1.toString();
+ }
+
+ }
+
+ @Override
+ public void close() {
+ if (!window.miniBatchEnd()) {
+ callUserFunctionAndLogException();
+ }
+ super.close();
+ }
+
+ @Override
+ protected void callUserFunction1() throws Exception {
+ }
+
+ @Override
+ protected void callUserFunction2() throws Exception {
+ }
+
+ public void setSlideSize(long slideSize) {
+ this.slideSize = slideSize;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecord.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecord.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecord.java
new file mode 100644
index 0000000..76f37be
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecord.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamrecord;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple;
+
+/**
+ * Object for wrapping a tuple or other object with ID used for sending records
+ * between streaming task in Apache Flink stream processing.
+ */
+public class StreamRecord<T> implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private UID uid;
+ private T streamObject;
+ public boolean isTuple;
+
+ /**
+ * Creates an empty StreamRecord
+ */
+ public StreamRecord() {
+ uid = new UID();
+ }
+
+ /**
+ * @return The ID of the object
+ */
+ public UID getId() {
+ return uid;
+ }
+
+ /**
+ * Creates a new ID for the StreamRecord using the given channelID
+ *
+ * @param channelID
+ * ID of the emitting task
+ * @return The StreamRecord object
+ */
+ public StreamRecord<T> newId(int channelID) {
+ uid = new UID(channelID);
+ return this;
+ }
+
+ /**
+ * Sets the ID of the StreamRecord
+ *
+ * @param id
+ * id to set
+ */
+ public void setId(UID id) {
+ this.uid = id;
+ }
+
+ /**
+ * Gets the wrapped object from the StreamRecord
+ *
+ * @return The object wrapped
+ */
+ public T getObject() {
+ return streamObject;
+ }
+
+ /**
+ * Gets the field of the contained object at the given position. If a tuple
+ * is wrapped then the getField method is invoked. If the StreamRecord
+ * contains and object of Basic types only position 0 could be returned.
+ *
+ * @param pos
+ * Position of the field to get.
+ * @return Returns the object contained in the position.
+ */
+ public Object getField(int pos) {
+ if (isTuple) {
+ return ((Tuple) streamObject).getField(pos);
+ } else {
+ if (pos == 0) {
+ return streamObject;
+ } else {
+ throw new IndexOutOfBoundsException();
+ }
+ }
+ }
+
+ /**
+ * Extracts key for the stored object using the keySelector provided.
+ *
+ * @param keySelector
+ * KeySelector for extracting the key
+ * @return The extracted key
+ */
+ public <R> R getKey(KeySelector<T, R> keySelector) {
+ try {
+ return keySelector.getKey(streamObject);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to extract key: " + e.getMessage());
+ }
+ }
+
+ /**
+ * Sets the object stored
+ *
+ * @param object
+ * Object to set
+ * @return Returns the StreamRecord object
+ */
+ public StreamRecord<T> setObject(T object) {
+ this.streamObject = object;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return streamObject.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecordSerializer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecordSerializer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecordSerializer.java
new file mode 100755
index 0000000..85faa9e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecordSerializer.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamrecord;
+
+import java.io.IOException;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+public final class StreamRecordSerializer<T> extends TypeSerializer<StreamRecord<T>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final TypeSerializer<T> typeSerializer;
+ private final boolean isTuple;
+
+ public StreamRecordSerializer(TypeInformation<T> typeInfo) {
+ this.typeSerializer = typeInfo.createSerializer();
+ this.isTuple = typeInfo.isTupleType();
+ }
+
+ public TypeSerializer<T> getObjectSerializer() {
+ return typeSerializer;
+ }
+
+ @Override
+ public boolean isImmutableType() {
+ return false;
+ }
+
+ @Override
+ public boolean isStateful() {
+ return false;
+ }
+
+ @Override
+ public StreamRecord<T> createInstance() {
+ try {
+ StreamRecord<T> t = new StreamRecord<T>();
+ t.isTuple = isTuple;
+ t.setObject(typeSerializer.createInstance());
+ return t;
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot instantiate StreamRecord.", e);
+ }
+ }
+
+ @Override
+ public StreamRecord<T> copy(StreamRecord<T> from) {
+ StreamRecord<T> rec = new StreamRecord<T>();
+ rec.isTuple = from.isTuple;
+ rec.setId(from.getId().copy());
+ rec.setObject(typeSerializer.copy(from.getObject()));
+ return rec;
+ }
+
+ @Override
+ public StreamRecord<T> copy(StreamRecord<T> from, StreamRecord<T> reuse) {
+ reuse.isTuple = from.isTuple;
+ reuse.setId(from.getId().copy());
+ reuse.setObject(typeSerializer.copy(from.getObject(), reuse.getObject()));
+ return reuse;
+ }
+
+ @Override
+ public int getLength() {
+ return -1;
+ }
+
+ @Override
+ public void serialize(StreamRecord<T> value, DataOutputView target) throws IOException {
+ value.getId().write(target);
+ typeSerializer.serialize(value.getObject(), target);
+ }
+
+ @Override
+ public StreamRecord<T> deserialize(DataInputView source) throws IOException {
+ StreamRecord<T> record = new StreamRecord<T>();
+ record.isTuple = this.isTuple;
+ record.getId().read(source);
+ record.setObject(typeSerializer.deserialize(source));
+ return record;
+ }
+
+ @Override
+ public StreamRecord<T> deserialize(StreamRecord<T> reuse, DataInputView source) throws IOException {
+ reuse.getId().read(source);
+ reuse.setObject(typeSerializer.deserialize(reuse.getObject(), source));
+ return reuse;
+ }
+
+ @Override
+ public void copy(DataInputView source, DataOutputView target) throws IOException {
+ // Needs to be implemented
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/UID.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/UID.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/UID.java
new file mode 100644
index 0000000..b79b7e5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/UID.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamrecord;
+
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Random;
+
+import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+/**
+ * Object for creating unique IDs for {@link StreamRecord}s.
+ *
+ **/
+public class UID implements IOReadableWritable, Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private ByteBuffer uid;
+ private static Random random = new Random();
+
+ public UID() {
+ uid = ByteBuffer.allocate(20);
+ }
+
+ // TODO: consider sequential ids
+ public UID(int channelID) {
+ byte[] uuid = new byte[16];
+ random.nextBytes(uuid);
+ uid = ByteBuffer.allocate(20).putInt(channelID).put(uuid);
+ }
+
+ UID(byte[] id) {
+ uid = ByteBuffer.wrap(id);
+ }
+
+ public int getChannelId() {
+ uid.position(0);
+ return uid.getInt();
+ }
+
+ public byte[] getGeneratedId() {
+ uid.position(4);
+ return uid.slice().array();
+ }
+
+ public byte[] getId() {
+ uid.position(0);
+ return uid.array();
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ out.write(uid.array());
+ }
+
+ private void writeObject(ObjectOutputStream stream) throws IOException {
+ stream.write(uid.array());
+ }
+
+ private void readObject(java.io.ObjectInputStream stream) throws IOException,
+ ClassNotFoundException {
+ byte[] uidA = new byte[20];
+ stream.read(uidA);
+ uid = ByteBuffer.allocate(20).put(uidA);
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+ byte[] uidByteArray = new byte[20];
+ in.readFully(uidByteArray, 0, 20);
+ uid = ByteBuffer.wrap(uidByteArray);
+ }
+
+ @Override
+ public String toString() {
+ return getChannelId() + "-" + Long.toHexString(uid.getLong(4)) + "-"
+ + Long.toHexString(uid.getLong(12));
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(getId());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ } else {
+ try {
+ UID other = (UID) obj;
+ return Arrays.equals(this.getId(), other.getId());
+ } catch (ClassCastException e) {
+ return false;
+ }
+ }
+ }
+
+ public UID copy() {
+ return new UID(Arrays.copyOf(uid.array(), 20));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
new file mode 100644
index 0000000..2b650be
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+import org.apache.flink.runtime.io.network.api.reader.BufferReader;
+import org.apache.flink.runtime.io.network.api.reader.BufferReaderBase;
+import org.apache.flink.runtime.io.network.api.reader.UnionBufferReader;
+import org.apache.flink.runtime.plugable.DeserializationDelegate;
+import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.io.CoReaderIterator;
+import org.apache.flink.streaming.io.CoRecordReader;
+import org.apache.flink.util.MutableObjectIterator;
+
+import java.util.ArrayList;
+
+public class CoStreamVertex<IN1, IN2, OUT> extends StreamVertex<IN1, OUT> {
+
+ protected StreamRecordSerializer<IN1> inputDeserializer1 = null;
+ protected StreamRecordSerializer<IN2> inputDeserializer2 = null;
+
+ MutableObjectIterator<StreamRecord<IN1>> inputIter1;
+ MutableObjectIterator<StreamRecord<IN2>> inputIter2;
+
+ CoRecordReader<DeserializationDelegate<StreamRecord<IN1>>, DeserializationDelegate<StreamRecord<IN2>>> coReader;
+ CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>> coIter;
+
+ private CoInvokable<IN1, IN2, OUT> userInvokable;
+ private static int numTasks;
+
+ public CoStreamVertex() {
+ userInvokable = null;
+ numTasks = newVertex();
+ instanceID = numTasks;
+ }
+
+ private void setDeserializers() {
+ inputDeserializer1 = configuration.getTypeSerializerIn1(userClassLoader);
+ inputDeserializer2 = configuration.getTypeSerializerIn2(userClassLoader);
+ }
+
+ @Override
+ public void setInputsOutputs() {
+ outputHandler = new OutputHandler<OUT>(this);
+
+ setConfigInputs();
+
+ coIter = new CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>>(coReader,
+ inputDeserializer1, inputDeserializer2);
+ }
+
+ @Override
+ protected void setInvokable() {
+ userInvokable = configuration.getUserInvokable(userClassLoader);
+ userInvokable.setup(this);
+ }
+
+ protected void setConfigInputs() throws StreamVertexException {
+ setDeserializers();
+
+ int numberOfInputs = configuration.getNumberOfInputs();
+
+ ArrayList<BufferReader> inputList1 = new ArrayList<BufferReader>();
+ ArrayList<BufferReader> inputList2 = new ArrayList<BufferReader>();
+
+ for (int i = 0; i < numberOfInputs; i++) {
+ int inputType = configuration.getInputIndex(i);
+ BufferReader reader = getEnvironment().getReader(i);
+ switch (inputType) {
+ case 1:
+ inputList1.add(reader);
+ break;
+ case 2:
+ inputList2.add(reader);
+ break;
+ default:
+ throw new RuntimeException("Invalid input type number: " + inputType);
+ }
+ }
+
+ final BufferReaderBase reader1 = inputList1.size() == 1
+ ? inputList1.get(0)
+ : new UnionBufferReader(inputList1.toArray(new BufferReader[inputList1.size()]));
+
+ final BufferReaderBase reader2 = inputList2.size() == 1
+ ? inputList2.get(0)
+ : new UnionBufferReader(inputList2.toArray(new BufferReader[inputList2.size()]));
+
+ coReader = new CoRecordReader<DeserializationDelegate<StreamRecord<IN1>>, DeserializationDelegate<StreamRecord<IN2>>>(reader1, reader2);
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ outputHandler.invokeUserFunction("CO-TASK", userInvokable);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <X> MutableObjectIterator<X> getInput(int index) {
+ switch (index) {
+ case 0:
+ return (MutableObjectIterator<X>) inputIter1;
+ case 1:
+ return (MutableObjectIterator<X>) inputIter2;
+ default:
+ throw new IllegalArgumentException("CoStreamVertex has only 2 inputs");
+ }
+ }
+ @SuppressWarnings("unchecked")
+ @Override
+ public <X> StreamRecordSerializer<X> getInputSerializer(int index) {
+ switch (index) {
+ case 0:
+ return (StreamRecordSerializer<X>) inputDeserializer1;
+ case 1:
+ return (StreamRecordSerializer<X>) inputDeserializer2;
+ default:
+ throw new IllegalArgumentException("CoStreamVertex has only 2 inputs");
+ }
+ }
+ @SuppressWarnings("unchecked")
+ @Override
+ public <X, Y> CoReaderIterator<X, Y> getCoReader() {
+ return (CoReaderIterator<X, Y>) coIter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
new file mode 100644
index 0000000..ad6d948
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.runtime.io.network.api.reader.MutableReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableRecordReader;
+import org.apache.flink.runtime.io.network.api.reader.UnionBufferReader;
+import org.apache.flink.runtime.operators.util.ReaderIterator;
+import org.apache.flink.runtime.plugable.DeserializationDelegate;
+import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.util.MutableObjectIterator;
+
+public class InputHandler<IN> {
+ private StreamRecordSerializer<IN> inputSerializer = null;
+ private MutableObjectIterator<StreamRecord<IN>> inputIter;
+ private MutableReader<IOReadableWritable> inputs;
+
+ private StreamVertex<IN, ?> streamVertex;
+ private StreamConfig configuration;
+
+ public InputHandler(StreamVertex<IN, ?> streamComponent) {
+ this.streamVertex = streamComponent;
+ this.configuration = new StreamConfig(streamComponent.getTaskConfiguration());
+ try {
+ setConfigInputs();
+ } catch (Exception e) {
+ throw new StreamVertexException("Cannot register inputs for "
+ + getClass().getSimpleName(), e);
+ }
+
+ }
+
+ protected void setConfigInputs() throws StreamVertexException {
+ inputSerializer = configuration.getTypeSerializerIn1(streamVertex.userClassLoader);
+
+ int numberOfInputs = configuration.getNumberOfInputs();
+ if (numberOfInputs > 0) {
+
+ if (numberOfInputs < 2) {
+
+ inputs = new MutableRecordReader<IOReadableWritable>(streamVertex.getEnvironment().getReader(0));
+
+ } else {
+ UnionBufferReader reader = new UnionBufferReader(streamVertex.getEnvironment().getAllReaders());
+ inputs = new MutableRecordReader<IOReadableWritable>(reader);
+ }
+
+ inputIter = createInputIterator();
+ }
+ }
+
+ private MutableObjectIterator<StreamRecord<IN>> createInputIterator() {
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ final MutableObjectIterator<StreamRecord<IN>> iter = new ReaderIterator(inputs,
+ inputSerializer);
+ return iter;
+ }
+
+ protected static <T> MutableObjectIterator<StreamRecord<T>> staticCreateInputIterator(
+ MutableReader<?> inputReader, TypeSerializer<?> serializer) {
+
+ // generic data type serialization
+ @SuppressWarnings("unchecked")
+ MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ final MutableObjectIterator<StreamRecord<T>> iter = new ReaderIterator(reader, serializer);
+ return iter;
+ }
+
+ public StreamRecordSerializer<IN> getInputSerializer() {
+ return inputSerializer;
+ }
+
+ public MutableObjectIterator<StreamRecord<IN>> getInputIter() {
+ return inputIter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
new file mode 100644
index 0000000..1a12cb2
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.collector.CollectorWrapper;
+import org.apache.flink.streaming.api.collector.DirectedCollectorWrapper;
+import org.apache.flink.streaming.api.collector.StreamOutput;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.io.StreamRecordWriter;
+import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class OutputHandler<OUT> {
+ private static final Logger LOG = LoggerFactory.getLogger(OutputHandler.class);
+
+ private StreamVertex<?, OUT> vertex;
+ private StreamConfig configuration;
+ private ClassLoader cl;
+ private Collector<OUT> outerCollector;
+
+ public List<ChainableInvokable<?, ?>> chainedInvokables;
+
+ private Map<String, StreamOutput<?>> outputMap;
+ private Map<String, StreamConfig> chainedConfigs;
+ private List<Tuple2<String, String>> outEdgesInOrder;
+
+ public OutputHandler(StreamVertex<?, OUT> vertex) {
+
+ // Initialize some fields
+ this.vertex = vertex;
+ this.configuration = new StreamConfig(vertex.getTaskConfiguration());
+ this.chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
+ this.outputMap = new HashMap<String, StreamOutput<?>>();
+ this.cl = vertex.getUserCodeClassLoader();
+
+ // We read the chained configs, and the order of record writer
+ // registrations by outputname
+ this.chainedConfigs = configuration.getTransitiveChainedTaskConfigs(cl);
+ this.chainedConfigs.put(configuration.getTaskName(), configuration);
+
+ this.outEdgesInOrder = configuration.getOutEdgesInOrder(cl);
+
+ // We iterate through all the out edges from this job vertex and create
+ // a stream output
+ for (Tuple2<String, String> outEdge : outEdgesInOrder) {
+ StreamOutput<?> streamOutput = createStreamOutput(outEdge.f1,
+ chainedConfigs.get(outEdge.f0), outEdgesInOrder.indexOf(outEdge));
+ outputMap.put(outEdge.f1, streamOutput);
+ }
+
+ // We create the outer collector that will be passed to the first task
+ // in the chain
+ this.outerCollector = createChainedCollector(configuration);
+
+ }
+
+ public Collection<StreamOutput<?>> getOutputs() {
+ return outputMap.values();
+ }
+
+ /**
+ * This method builds up a nested collector which encapsulates all the
+ * chained operators and their network output. The result of this recursive
+ * call will be passed as collector to the first invokable in the chain.
+ *
+ * @param chainedTaskConfig
+ * The configuration of the starting operator of the chain, we
+ * use this paramater to recursively build the whole chain
+ * @return Returns the collector for the chain starting from the given
+ * config
+ */
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private Collector<OUT> createChainedCollector(StreamConfig chainedTaskConfig) {
+
+ boolean isDirectEmit = chainedTaskConfig.isDirectedEmit();
+
+ // We create a wrapper that will encapsulate the chained operators and
+ // network outputs
+ CollectorWrapper<OUT> wrapper = isDirectEmit ? new DirectedCollectorWrapper(
+ chainedTaskConfig.getOutputSelectors(cl)) : new CollectorWrapper<OUT>();
+
+ // Create collectors for the network outputs
+ for (String output : chainedTaskConfig.getOutputs(cl)) {
+
+ Collector<?> outCollector = outputMap.get(output);
+
+ if (isDirectEmit) {
+ ((DirectedCollectorWrapper<OUT>) wrapper).addCollector(outCollector,
+ chainedTaskConfig.getSelectedNames(output));
+ } else {
+ wrapper.addCollector(outCollector);
+ }
+ }
+
+ // Create collectors for the chained outputs
+ for (String output : chainedTaskConfig.getChainedOutputs(cl)) {
+ Collector<?> outCollector = createChainedCollector(chainedConfigs.get(output));
+ if (isDirectEmit) {
+ ((DirectedCollectorWrapper<OUT>) wrapper).addCollector(outCollector,
+ chainedTaskConfig.getSelectedNames(output));
+ } else {
+ wrapper.addCollector(outCollector);
+ }
+ }
+
+ if (chainedTaskConfig.isChainStart()) {
+ // The current task is the first chained task at this vertex so we
+ // return the wrapper
+ return wrapper;
+ } else {
+ // The current task is a part of the chain so we get the chainable
+ // invokable which will be returned and set it up using the wrapper
+ ChainableInvokable chainableInvokable = chainedTaskConfig.getUserInvokable(vertex
+ .getUserCodeClassLoader());
+ chainableInvokable.setup(wrapper,
+ chainedTaskConfig.getTypeSerializerIn1(vertex.getUserCodeClassLoader()));
+
+ chainedInvokables.add(chainableInvokable);
+ return chainableInvokable;
+ }
+
+ }
+
+ public Collector<OUT> getCollector() {
+ return outerCollector;
+ }
+
+ /**
+ * We create the StreamOutput for the specific output given by the name, and
+ * the configuration of its source task
+ *
+ * @param outputVertex
+ * Name of the output to which the streamoutput will be set up
+ * @param configuration
+ * The config of upStream task
+ * @return
+ */
+ private <T> StreamOutput<T> createStreamOutput(String outputVertex, StreamConfig configuration,
+ int outputIndex) {
+
+ StreamRecordSerializer<T> outSerializer = configuration
+ .getTypeSerializerOut1(vertex.userClassLoader);
+ SerializationDelegate<StreamRecord<T>> outSerializationDelegate = null;
+
+ if (outSerializer != null) {
+ outSerializationDelegate = new SerializationDelegate<StreamRecord<T>>(outSerializer);
+ outSerializationDelegate.setInstance(outSerializer.createInstance());
+ }
+
+ StreamPartitioner<T> outputPartitioner = configuration.getPartitioner(cl, outputVertex);
+
+ RecordWriter<SerializationDelegate<StreamRecord<T>>> output;
+
+ if (configuration.getBufferTimeout() >= 0) {
+
+ output = new StreamRecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
+ .getEnvironment().getWriter(outputIndex), outputPartitioner,
+ configuration.getBufferTimeout());
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("StreamRecordWriter initiated with {} bufferTimeout for {}",
+ configuration.getBufferTimeout(), vertex.getClass().getSimpleName());
+ }
+ } else {
+ output = new RecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
+ .getEnvironment().getWriter(outputIndex), outputPartitioner);
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("RecordWriter initiated for {}", vertex.getClass().getSimpleName());
+ }
+ }
+
+ StreamOutput<T> streamOutput = new StreamOutput<T>(output, vertex.instanceID,
+ outSerializationDelegate);
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
+ .getSimpleName(), outputIndex, vertex.getClass().getSimpleName());
+ }
+
+ return streamOutput;
+ }
+
+ public void flushOutputs() throws IOException, InterruptedException {
+ for (StreamOutput<?> streamOutput : getOutputs()) {
+ streamOutput.close();
+ }
+ }
+
+ public void invokeUserFunction(String componentTypeName, StreamInvokable<?, OUT> userInvokable)
+ throws IOException, InterruptedException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{} {} invoked with instance id {}", componentTypeName, vertex.getName(),
+ vertex.getInstanceID());
+ }
+
+ try {
+ vertex.invokeUserFunction(userInvokable);
+ } catch (Exception e) {
+ flushOutputs();
+ throw new RuntimeException(e);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{} {} invoke finished instance id {}", componentTypeName, vertex.getName(),
+ vertex.getInstanceID());
+ }
+
+ flushOutputs();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
new file mode 100755
index 0000000..bc89241
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.streaming.api.collector.StreamOutput;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.io.BlockingQueueBroker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StreamIterationHead<OUT> extends StreamVertex<OUT, OUT> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StreamIterationHead.class);
+
+ private OutputHandler<OUT> outputHandler;
+
+ private static int numSources;
+ private Integer iterationId;
+ @SuppressWarnings("rawtypes")
+ private BlockingQueue<StreamRecord> dataChannel;
+ private long iterationWaitTime;
+ private boolean shouldWait;
+
+ @SuppressWarnings("rawtypes")
+ public StreamIterationHead() {
+ numSources = newVertex();
+ instanceID = numSources;
+ dataChannel = new ArrayBlockingQueue<StreamRecord>(1);
+ }
+
+ @Override
+ public void setInputsOutputs() {
+ outputHandler = new OutputHandler<OUT>(this);
+
+ iterationId = configuration.getIterationId();
+ iterationWaitTime = configuration.getIterationWaitTime();
+ shouldWait = iterationWaitTime > 0;
+
+ try {
+ BlockingQueueBroker.instance().handIn(iterationId.toString(), dataChannel);
+ } catch (Exception e) {
+
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void invoke() throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SOURCE {} invoked with instance id {}", getName(), getInstanceID());
+ }
+
+ StreamRecord<OUT> nextRecord;
+
+ List<StreamOutput<OUT>> outputs = new LinkedList<StreamOutput<OUT>>();
+ for (StreamOutput<?> output : outputHandler.getOutputs()) {
+ outputs.add((StreamOutput<OUT>) output);
+ }
+
+ while (true) {
+ if (shouldWait) {
+ nextRecord = dataChannel.poll(iterationWaitTime, TimeUnit.MILLISECONDS);
+ } else {
+ nextRecord = dataChannel.take();
+ }
+ if (nextRecord == null) {
+ break;
+ }
+ for (StreamOutput<OUT> output : outputs) {
+ output.collect(nextRecord.getObject());
+ }
+ }
+
+ outputHandler.flushOutputs();
+ }
+
+ @Override
+ protected void setInvokable() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
new file mode 100755
index 0000000..b3ecdf8
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.io.BlockingQueueBroker;
+import org.apache.flink.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StreamIterationTail<IN> extends StreamVertex<IN, IN> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StreamIterationTail.class);
+
+ private InputHandler<IN> inputHandler;
+
+ private Integer iterationId;
+ @SuppressWarnings("rawtypes")
+ private BlockingQueue<StreamRecord> dataChannel;
+ private long iterationWaitTime;
+ private boolean shouldWait;
+
+ public StreamIterationTail() {
+ }
+
+ @Override
+ public void setInputsOutputs() {
+ try {
+ inputHandler = new InputHandler<IN>(this);
+
+ iterationId = configuration.getIterationId();
+ iterationWaitTime = configuration.getIterationWaitTime();
+ shouldWait = iterationWaitTime > 0;
+ dataChannel = BlockingQueueBroker.instance().get(iterationId.toString());
+ } catch (Exception e) {
+ throw new StreamVertexException(String.format(
+ "Cannot register inputs of StreamIterationSink %s", iterationId), e);
+ }
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SINK {} invoked", getName());
+ }
+
+ forwardRecords();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SINK {} invoke finished", getName());
+ }
+ }
+
+ protected void forwardRecords() throws Exception {
+ StreamRecord<IN> reuse = inputHandler.getInputSerializer().createInstance();
+ while ((reuse = inputHandler.getInputIter().next(reuse)) != null) {
+ if (!pushToQueue(reuse)) {
+ break;
+ }
+ // TODO: Fix object reuse for iteration
+ reuse = inputHandler.getInputSerializer().createInstance();
+ }
+ }
+
+ private boolean pushToQueue(StreamRecord<IN> record) {
+ try {
+ if (shouldWait) {
+ return dataChannel.offer(record, iterationWaitTime, TimeUnit.MILLISECONDS);
+ } else {
+ dataChannel.put(record);
+ return true;
+ }
+ } catch (InterruptedException e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Pushing back record at iteration %s failed due to: {}", iterationId,
+ StringUtils.stringifyException(e));
+ }
+ return false;
+ }
+ }
+
+ @Override
+ protected void setInvokable() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java
new file mode 100644
index 0000000..7fbab3b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.io.CoReaderIterator;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.MutableObjectIterator;
+
+public interface StreamTaskContext<OUT> {
+
+ StreamConfig getConfig();
+
+ ClassLoader getUserCodeClassLoader();
+
+ <X> MutableObjectIterator<X> getInput(int index);
+
+ <X> StreamRecordSerializer<X> getInputSerializer(int index);
+
+ Collector<OUT> getOutputCollector();
+
+ <X, Y> CoReaderIterator<X, Y> getCoReader();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
new file mode 100644
index 0000000..994b1fa
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+import java.util.Map;
+
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.io.CoReaderIterator;
+import org.apache.flink.streaming.state.OperatorState;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.MutableObjectIterator;
+
+public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTaskContext<OUT> {
+
+ private static int numTasks;
+
+ protected StreamConfig configuration;
+ protected int instanceID;
+ private static int numVertices = 0;
+
+ private InputHandler<IN> inputHandler;
+ protected OutputHandler<OUT> outputHandler;
+ private StreamInvokable<IN, OUT> userInvokable;
+
+ private StreamingRuntimeContext context;
+ private Map<String, OperatorState<?>> states;
+
+ protected ClassLoader userClassLoader;
+
+ public StreamVertex() {
+ userInvokable = null;
+ numTasks = newVertex();
+ instanceID = numTasks;
+ }
+
+ protected static int newVertex() {
+ numVertices++;
+ return numVertices;
+ }
+
+ @Override
+ public void registerInputOutput() {
+ initialize();
+ setInputsOutputs();
+ setInvokable();
+ }
+
+ protected void initialize() {
+ this.userClassLoader = getUserCodeClassLoader();
+ this.configuration = new StreamConfig(getTaskConfiguration());
+ this.states = configuration.getOperatorStates(userClassLoader);
+ this.context = createRuntimeContext(getEnvironment().getTaskName(), this.states);
+ }
+
+ protected <T> void invokeUserFunction(StreamInvokable<?, T> userInvokable) throws Exception {
+ userInvokable.setRuntimeContext(context);
+ userInvokable.open(getTaskConfiguration());
+
+ for (ChainableInvokable<?, ?> invokable : outputHandler.chainedInvokables) {
+ invokable.setRuntimeContext(context);
+ invokable.open(getTaskConfiguration());
+ }
+
+ userInvokable.invoke();
+ userInvokable.close();
+
+ for (ChainableInvokable<?, ?> invokable : outputHandler.chainedInvokables) {
+ invokable.close();
+ }
+
+ }
+
+ public void setInputsOutputs() {
+ inputHandler = new InputHandler<IN>(this);
+ outputHandler = new OutputHandler<OUT>(this);
+ }
+
+ protected void setInvokable() {
+ userInvokable = configuration.getUserInvokable(userClassLoader);
+ userInvokable.setup(this);
+ }
+
+ public String getName() {
+ return getEnvironment().getTaskName();
+ }
+
+ public int getInstanceID() {
+ return instanceID;
+ }
+
+ public StreamingRuntimeContext createRuntimeContext(String taskName,
+ Map<String, OperatorState<?>> states) {
+ Environment env = getEnvironment();
+ return new StreamingRuntimeContext(taskName, env, getUserCodeClassLoader(), states);
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ outputHandler.invokeUserFunction("TASK", userInvokable);
+ }
+
+ @Override
+ public StreamConfig getConfig() {
+ return configuration;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <X> MutableObjectIterator<X> getInput(int index) {
+ if (index == 0) {
+ return (MutableObjectIterator<X>) inputHandler.getInputIter();
+ } else {
+ throw new IllegalArgumentException("There is only 1 input");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <X> StreamRecordSerializer<X> getInputSerializer(int index) {
+ if (index == 0) {
+ return (StreamRecordSerializer<X>) inputHandler.getInputSerializer();
+ } else {
+ throw new IllegalArgumentException("There is only 1 input");
+ }
+ }
+
+ @Override
+ public Collector<OUT> getOutputCollector() {
+ return outputHandler.getCollector();
+ }
+
+ @Override
+ public <X, Y> CoReaderIterator<X, Y> getCoReader() {
+ throw new IllegalArgumentException("CoReader not available");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertexException.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertexException.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertexException.java
new file mode 100644
index 0000000..ed8b91e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertexException.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+/**
+ * An exception that is thrown by the stream verices when encountering an
+ * illegal condition.
+ */
+public class StreamVertexException extends RuntimeException {
+
+ /**
+ * Serial version UID for serialization interoperability.
+ */
+ private static final long serialVersionUID = 8392043527067472439L;
+
+ /**
+ * Creates a compiler exception with no message and no cause.
+ */
+ public StreamVertexException() {
+ }
+
+ /**
+ * Creates a compiler exception with the given message and no cause.
+ *
+ * @param message
+ * The message for the exception.
+ */
+ public StreamVertexException(String message) {
+ super(message);
+ }
+
+ /**
+ * Creates a compiler exception with the given cause and no message.
+ *
+ * @param cause
+ * The <tt>Throwable</tt> that caused this exception.
+ */
+ public StreamVertexException(Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * Creates a compiler exception with the given message and cause.
+ *
+ * @param message
+ * The message for the exception.
+ * @param cause
+ * The <tt>Throwable</tt> that caused this exception.
+ */
+ public StreamVertexException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamingRuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamingRuntimeContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamingRuntimeContext.java
new file mode 100644
index 0000000..a1a64e2
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamingRuntimeContext.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+import java.util.Map;
+
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.functions.util.RuntimeUDFContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
+import org.apache.flink.runtime.operators.util.TaskConfig;
+import org.apache.flink.streaming.state.OperatorState;
+
+/**
+ * Implementation of the {@link RuntimeContext}, created by runtime stream UDF
+ * operators.
+ */
+public class StreamingRuntimeContext extends RuntimeUDFContext {
+
+ public Environment env;
+ private final Map<String, OperatorState<?>> operatorStates;
+
+ public StreamingRuntimeContext(String name, Environment env, ClassLoader userCodeClassLoader,
+ Map<String, OperatorState<?>> operatorStates) {
+ super(name, env.getNumberOfSubtasks(), env.getIndexInSubtaskGroup(),
+ userCodeClassLoader, env.getCopyTask());
+ this.env = env;
+ this.operatorStates = operatorStates;
+ }
+
+ /**
+ * Returns the operator state registered by the given name for the operator.
+ *
+ * @param name
+ * Name of the operator state to be returned.
+ * @return The operator state.
+ */
+ public OperatorState<?> getState(String name) {
+ if (operatorStates == null) {
+ throw new RuntimeException("No state has been registered for this operator.");
+ } else {
+ OperatorState<?> state = operatorStates.get(name);
+ if (state != null) {
+ return state;
+ } else {
+ throw new RuntimeException("No state has been registered for the name: " + name);
+ }
+ }
+
+ }
+
+ /**
+ * Returns the input split provider associated with the operator.
+ *
+ * @return The input split provider.
+ */
+ public InputSplitProvider getInputSplitProvider() {
+ return env.getInputSplitProvider();
+ }
+
+ /**
+ * Returns the stub parameters associated with the {@link TaskConfig} of the
+ * operator.
+ *
+ * @return The stub parameters.
+ */
+ public Configuration getTaskStubParameters() {
+ return new TaskConfig(env.getTaskConfiguration()).getStubParameters();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java
new file mode 100644
index 0000000..15aaf51
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+
+/**
+ * This delta function calculates the cosine distance between two given vectors.
+ * The cosine distance is defined as: cosineDistance=1-cosineSimilarity
+ *
+ * Cosine similarity: http://en.wikipedia.org/wiki/Cosine_similarity
+ *
+ * @param <DATA>
+ * The input data type. This delta function works with a double[],
+ * but can extract/convert to it from any other given object in case
+ * the respective extractor has been set. See
+ * {@link ExtractionAwareDeltaFunction} for more information.
+ */
+public class CosineDistance<DATA> extends ExtractionAwareDeltaFunction<DATA, double[]> {
+
+ /**
+ * auto-generated id
+ */
+ private static final long serialVersionUID = -1217813582965151599L;
+
+ public CosineDistance() {
+ super(null);
+ }
+
+ public CosineDistance(Extractor<DATA, double[]> converter) {
+ super(converter);
+ }
+
+ @Override
+ public double getNestedDelta(double[] oldDataPoint, double[] newDataPoint) {
+ if (isNullvector(oldDataPoint, newDataPoint)) {
+ return 0;
+ }
+
+ double sum1 = 0;
+ double sum2 = 0;
+ for (int i = 0; i < oldDataPoint.length; i++) {
+ sum1 += oldDataPoint[i] * oldDataPoint[i];
+ sum2 += newDataPoint[i] * newDataPoint[i];
+ }
+ sum1 = Math.sqrt(sum1);
+ sum2 = Math.sqrt(sum2);
+
+ return 1d - (dotProduct(oldDataPoint, newDataPoint) / (sum1 * sum2));
+ }
+
+ private double dotProduct(double[] a, double[] b) {
+ double result = 0;
+ for (int i = 0; i < a.length; i++) {
+ result += a[i] * b[i];
+ }
+ return result;
+ }
+
+ private boolean isNullvector(double[]... vectors) {
+ outer: for (double[] v : vectors) {
+ for (double field : v) {
+ if (field != 0) {
+ continue outer;
+ }
+ }
+ // This position is only reached in case all fields are 0.
+ return true;
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java
new file mode 100644
index 0000000..b2223d6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import java.io.Serializable;
+
+/**
+ * This interface allows the implementation of a function which calculates the
+ * delta between two data points. Delta functions might be used in delta
+ * policies and allow flexible adaptive windowing based on the arriving data
+ * points.
+ *
+ * @param <DATA>
+ * The type of input data which can be compared using this function.
+ */
+public interface DeltaFunction<DATA> extends Serializable {
+
+ /**
+ * Calculates the delta between two given data points.
+ *
+ * @param oldDataPoint
+ * the old data point.
+ * @param newDataPoint
+ * the new data point.
+ * @return the delta between the two given points.
+ */
+ public double getDelta(DATA oldDataPoint, DATA newDataPoint);
+
+}
[13/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java
new file mode 100644
index 0000000..1c273d3
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.aggregation;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+
+public abstract class AggregationFunction<T> implements ReduceFunction<T> {
+ private static final long serialVersionUID = 1L;
+
+ public int position;
+
+ public AggregationFunction(int pos) {
+ this.position = pos;
+ }
+
+ public static enum AggregationType {
+ SUM, MIN, MAX, MINBY, MAXBY,
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
new file mode 100644
index 0000000..226c45a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.aggregation;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
+import java.util.List;
+
+import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.runtime.PojoComparator;
+
+public abstract class ComparableAggregator<T> extends AggregationFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ public Comparator comparator;
+ public boolean byAggregate;
+ public boolean first;
+
+ public ComparableAggregator(int pos, AggregationType aggregationType, boolean first) {
+ super(pos);
+ this.comparator = Comparator.getForAggregation(aggregationType);
+ this.byAggregate = (aggregationType == AggregationType.MAXBY)
+ || (aggregationType == AggregationType.MINBY);
+ this.first = first;
+ }
+
+ public static <R> AggregationFunction<R> getAggregator(int positionToAggregate,
+ TypeInformation<R> typeInfo, AggregationType aggregationType) {
+ return getAggregator(positionToAggregate, typeInfo, aggregationType, false);
+ }
+
+ public static <R> AggregationFunction<R> getAggregator(int positionToAggregate,
+ TypeInformation<R> typeInfo, AggregationType aggregationType, boolean first) {
+
+ if (typeInfo.isTupleType()) {
+ return new TupleComparableAggregator<R>(positionToAggregate, aggregationType, first);
+ } else if (typeInfo instanceof BasicArrayTypeInfo
+ || typeInfo instanceof PrimitiveArrayTypeInfo) {
+ return new ArrayComparableAggregator<R>(positionToAggregate, aggregationType, first);
+ } else {
+ return new SimpleComparableAggregator<R>(aggregationType);
+ }
+ }
+
+ public static <R> AggregationFunction<R> getAggregator(String field,
+ TypeInformation<R> typeInfo, AggregationType aggregationType, boolean first) {
+
+ return new PojoComparableAggregator<R>(field, typeInfo, aggregationType, first);
+ }
+
+ private static class TupleComparableAggregator<T> extends ComparableAggregator<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ public TupleComparableAggregator(int pos, AggregationType aggregationType, boolean first) {
+ super(pos, aggregationType, first);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T reduce(T value1, T value2) throws Exception {
+ Tuple tuple1 = (Tuple) value1;
+ Tuple tuple2 = (Tuple) value2;
+
+ Comparable<Object> o1 = tuple1.getField(position);
+ Object o2 = tuple2.getField(position);
+
+ int c = comparator.isExtremal(o1, o2);
+
+ if (byAggregate) {
+ if (c == 1) {
+ return (T) tuple1;
+ }
+ if (first) {
+ if (c == 0) {
+ return (T) tuple1;
+ }
+ }
+
+ return (T) tuple2;
+
+ } else {
+ if (c == 1) {
+ tuple2.setField(o1, position);
+ }
+ return (T) tuple2;
+ }
+
+ }
+ }
+
+ private static class ArrayComparableAggregator<T> extends ComparableAggregator<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ public ArrayComparableAggregator(int pos, AggregationType aggregationType, boolean first) {
+ super(pos, aggregationType, first);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T reduce(T array1, T array2) throws Exception {
+
+ Object v1 = Array.get(array1, position);
+ Object v2 = Array.get(array2, position);
+
+ int c = comparator.isExtremal((Comparable<Object>) v1, v2);
+
+ if (byAggregate) {
+ if (c == 1) {
+ return array1;
+ }
+ if (first) {
+ if (c == 0) {
+ return array1;
+ }
+ }
+
+ return array2;
+ } else {
+ if (c == 1) {
+ Array.set(array2, position, v1);
+ }
+
+ return array2;
+ }
+ }
+
+ }
+
+ private static class SimpleComparableAggregator<T> extends ComparableAggregator<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ public SimpleComparableAggregator(AggregationType aggregationType) {
+ super(0, aggregationType, false);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T reduce(T value1, T value2) throws Exception {
+
+ if (comparator.isExtremal((Comparable<Object>) value1, value2) == 1) {
+ return value1;
+ } else {
+ return value2;
+ }
+ }
+
+ }
+
+ private static class PojoComparableAggregator<T> extends ComparableAggregator<T> {
+
+ private static final long serialVersionUID = 1L;
+ PojoComparator<T> pojoComparator;
+
+ public PojoComparableAggregator(String field, TypeInformation<?> typeInfo,
+ AggregationType aggregationType, boolean first) {
+ super(0, aggregationType, first);
+ if (!(typeInfo instanceof CompositeType<?>)) {
+ throw new IllegalArgumentException(
+ "Key expressions are only supported on POJO types and Tuples. "
+ + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
+ }
+
+ @SuppressWarnings("unchecked")
+ CompositeType<T> cType = (CompositeType<T>) typeInfo;
+
+ List<FlatFieldDescriptor> fieldDescriptors = cType.getFlatFields(field);
+ int logicalKeyPosition = fieldDescriptors.get(0).getPosition();
+
+ if (cType instanceof PojoTypeInfo) {
+ pojoComparator = (PojoComparator<T>) cType.createComparator(
+ new int[] { logicalKeyPosition }, new boolean[] { false }, 0);
+ } else {
+ throw new IllegalArgumentException(
+ "Key expressions are only supported on POJO types. "
+ + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
+ }
+ }
+
+ @Override
+ public T reduce(T value1, T value2) throws Exception {
+
+ Field[] keyFields = pojoComparator.getKeyFields();
+ Object field1 = pojoComparator.accessField(keyFields[0], value1);
+ Object field2 = pojoComparator.accessField(keyFields[0], value2);
+
+ @SuppressWarnings("unchecked")
+ int c = comparator.isExtremal((Comparable<Object>) field1, field2);
+
+ if (byAggregate) {
+ if (c == 1) {
+ return value1;
+ }
+ if (first) {
+ if (c == 0) {
+ return value1;
+ }
+ }
+
+ return value2;
+ } else {
+ if (c == 1) {
+ keyFields[0].set(value2, field1);
+ }
+
+ return value2;
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/Comparator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/Comparator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/Comparator.java
new file mode 100644
index 0000000..f56774b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/Comparator.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.aggregation;
+
+import java.io.Serializable;
+
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
+
+public abstract class Comparator implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public abstract <R> int isExtremal(Comparable<R> o1, R o2);
+
+ public static Comparator getForAggregation(AggregationType type) {
+ switch (type) {
+ case MAX:
+ return new MaxComparator();
+ case MIN:
+ return new MinComparator();
+ case MINBY:
+ return new MinByComparator();
+ case MAXBY:
+ return new MaxByComparator();
+ default:
+ throw new IllegalArgumentException("Unsupported aggregation type.");
+ }
+ }
+
+ private static class MaxComparator extends Comparator {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public <R> int isExtremal(Comparable<R> o1, R o2) {
+ return o1.compareTo(o2) > 0 ? 1 : 0;
+ }
+
+ }
+
+ private static class MaxByComparator extends Comparator {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public <R> int isExtremal(Comparable<R> o1, R o2) {
+ int c = o1.compareTo(o2);
+ if (c > 0) {
+ return 1;
+ }
+ if (c == 0) {
+ return 0;
+ } else {
+ return -1;
+ }
+ }
+
+ }
+
+ private static class MinByComparator extends Comparator {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public <R> int isExtremal(Comparable<R> o1, R o2) {
+ int c = o1.compareTo(o2);
+ if (c < 0) {
+ return 1;
+ }
+ if (c == 0) {
+ return 0;
+ } else {
+ return -1;
+ }
+ }
+
+ }
+
+ private static class MinComparator extends Comparator {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public <R> int isExtremal(Comparable<R> o1, R o2) {
+ return o1.compareTo(o2) < 0 ? 1 : 0;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java
new file mode 100644
index 0000000..142028b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.aggregation;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.runtime.PojoComparator;
+
+public abstract class SumAggregator {
+
+ public static <T> ReduceFunction<T> getSumFunction(int pos, Class<?> clazz,
+ TypeInformation<T> typeInfo) {
+
+ if (typeInfo.isTupleType()) {
+ return new TupleSumAggregator<T>(pos, SumFunction.getForClass(clazz));
+ } else if (typeInfo instanceof BasicArrayTypeInfo
+ || typeInfo instanceof PrimitiveArrayTypeInfo) {
+ return new ArraySumAggregator<T>(pos, SumFunction.getForClass(clazz));
+ } else {
+ return new SimpleSumAggregator<T>(SumFunction.getForClass(clazz));
+ }
+
+ }
+
+ public static <T> ReduceFunction<T> getSumFunction(String field, TypeInformation<T> typeInfo) {
+
+ return new PojoSumAggregator<T>(field, typeInfo);
+ }
+
+ private static class TupleSumAggregator<T> extends AggregationFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ SumFunction adder;
+
+ public TupleSumAggregator(int pos, SumFunction adder) {
+ super(pos);
+ this.adder = adder;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T reduce(T value1, T value2) throws Exception {
+
+ Tuple tuple1 = (Tuple) value1;
+ Tuple tuple2 = (Tuple) value2;
+
+ tuple2.setField(adder.add(tuple1.getField(position), tuple2.getField(position)),
+ position);
+
+ return (T) tuple2;
+ }
+
+ }
+
+ private static class ArraySumAggregator<T> extends AggregationFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ SumFunction adder;
+
+ public ArraySumAggregator(int pos, SumFunction adder) {
+ super(pos);
+ this.adder = adder;
+ }
+
+ @Override
+ public T reduce(T value1, T value2) throws Exception {
+
+ Object v1 = Array.get(value1, position);
+ Object v2 = Array.get(value2, position);
+ Array.set(value2, position, adder.add(v1, v2));
+ return value2;
+ }
+
+ }
+
+ private static class SimpleSumAggregator<T> extends AggregationFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ SumFunction adder;
+
+ public SimpleSumAggregator(SumFunction adder) {
+ super(0);
+ this.adder = adder;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T reduce(T value1, T value2) throws Exception {
+
+ return (T) adder.add(value1, value2);
+ }
+
+ }
+
+ private static class PojoSumAggregator<T> extends AggregationFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+ SumFunction adder;
+ PojoComparator<T> comparator;
+
+ public PojoSumAggregator(String field, TypeInformation<?> type) {
+ super(0);
+ if (!(type instanceof CompositeType<?>)) {
+ throw new IllegalArgumentException(
+ "Key expressions are only supported on POJO types and Tuples. "
+ + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
+ }
+
+ @SuppressWarnings("unchecked")
+ CompositeType<T> cType = (CompositeType<T>) type;
+
+ List<FlatFieldDescriptor> fieldDescriptors = cType.getFlatFields(field);
+
+ int logicalKeyPosition = fieldDescriptors.get(0).getPosition();
+ Class<?> keyClass = fieldDescriptors.get(0).getType().getTypeClass();
+
+ adder = SumFunction.getForClass(keyClass);
+
+ if (cType instanceof PojoTypeInfo) {
+ comparator = (PojoComparator<T>) cType.createComparator(
+ new int[] { logicalKeyPosition }, new boolean[] { false }, 0);
+ } else {
+ throw new IllegalArgumentException(
+ "Key expressions are only supported on POJO types. "
+ + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
+ }
+ }
+
+ @Override
+ public T reduce(T value1, T value2) throws Exception {
+
+ Field[] keyFields = comparator.getKeyFields();
+ Object field1 = comparator.accessField(keyFields[0], value1);
+ Object field2 = comparator.accessField(keyFields[0], value2);
+
+ keyFields[0].set(value2, adder.add(field1, field2));
+
+ return value2;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java
new file mode 100644
index 0000000..2aef19c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.aggregation;
+
+import java.io.Serializable;
+
+public abstract class SumFunction implements Serializable{
+
+ private static final long serialVersionUID = 1L;
+
+ public abstract Object add(Object o1, Object o2);
+
+ public static SumFunction getForClass(Class<?> clazz) {
+
+ if (clazz == Integer.class) {
+ return new IntSum();
+ } else if (clazz == Long.class) {
+ return new LongSum();
+ } else if (clazz == Short.class) {
+ return new ShortSum();
+ } else if (clazz == Double.class) {
+ return new DoubleSum();
+ } else if (clazz == Float.class) {
+ return new FloatSum();
+ } else if (clazz == Byte.class) {
+ return new ByteSum();
+ } else {
+ throw new RuntimeException("DataStream cannot be summed because the class "
+ + clazz.getSimpleName() + " does not support the + operator.");
+ }
+ }
+
+ public static class IntSum extends SumFunction {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Object add(Object value1, Object value2) {
+ return (Integer) value1 + (Integer) value2;
+ }
+ }
+
+ public static class LongSum extends SumFunction {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Object add(Object value1, Object value2) {
+ return (Long) value1 + (Long) value2;
+ }
+ }
+
+ public static class DoubleSum extends SumFunction {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Object add(Object value1, Object value2) {
+ return (Double) value1 + (Double) value2;
+ }
+ }
+
+ public static class ShortSum extends SumFunction {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Object add(Object value1, Object value2) {
+ return (Short) value1 + (Short) value2;
+ }
+ }
+
+ public static class FloatSum extends SumFunction {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Object add(Object value1, Object value2) {
+ return (Float) value1 + (Float) value2;
+ }
+ }
+
+ public static class ByteSum extends SumFunction {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Object add(Object value1, Object value2) {
+ return (Byte) value1 + (Byte) value2;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoFlatMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoFlatMapFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoFlatMapFunction.java
new file mode 100644
index 0000000..10e8bac
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoFlatMapFunction.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.co;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.util.Collector;
+
+/**
+ * A CoFlatMapFunction represents a FlatMap transformation with two different
+ * input types.
+ *
+ * @param <IN1>
+ * Type of the first input.
+ * @param <IN2>
+ * Type of the second input.
+ * @param <OUT>
+ * Output type.
+ */
+public interface CoFlatMapFunction<IN1, IN2, OUT> extends Function, Serializable {
+
+ public void flatMap1(IN1 value, Collector<OUT> out) throws Exception;
+
+ public void flatMap2(IN2 value, Collector<OUT> out) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoMapFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoMapFunction.java
new file mode 100644
index 0000000..2903828
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoMapFunction.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.co;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.Function;
+
+/**
+ * A CoMapFunction represents a Map transformation with two different input
+ * types.
+ *
+ * @param <IN1>
+ * Type of the first input.
+ * @param <IN2>
+ * Type of the second input.
+ * @param <OUT>
+ * Output type.
+ */
+public interface CoMapFunction<IN1, IN2, OUT> extends Function, Serializable {
+
+ public OUT map1(IN1 value);
+
+ public OUT map2(IN2 value);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoReduceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoReduceFunction.java
new file mode 100644
index 0000000..879a1b4
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoReduceFunction.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.co;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.Function;
+
+/**
+ * The CoReduceFunction interface represents a Reduce transformation with two
+ * different input streams. The reduce1 function combine groups of elements of
+ * the first input with the same key to a single value, while reduce2 combine
+ * groups of elements of the second input with the same key to a single value.
+ * Each produced values are mapped to the same type by map1 and map2,
+ * respectively, to form one output stream.
+ *
+ * The basic syntax for using a grouped ReduceFunction is as follows:
+ *
+ * <pre>
+ * <blockquote>
+ * ConnectedDataStream<X> input = ...;
+ *
+ * ConnectedDataStream<X> result = input.groupBy(keyPosition1, keyPosition2)
+ * .reduce(new MyCoReduceFunction(), keyPosition1, keyPosition2).addSink(...);
+ * </blockquote>
+ * </pre>
+ * <p>
+ *
+ * @param <IN1>
+ * Type of the first input.
+ * @param <IN2>
+ * Type of the second input.
+ * @param <OUT>
+ * Output type.
+ */
+public interface CoReduceFunction<IN1, IN2, OUT> extends Function, Serializable {
+
+ /**
+ * The core method of CoReduceFunction, combining two values of the first
+ * input into one value of the same type. The reduce1 function is
+ * consecutively applied to all values of a group until only a single value
+ * remains.
+ *
+ * @param value1
+ * The first value to combine.
+ * @param value2
+ * The second value to combine.
+ * @return The combined value of both input values.
+ *
+ * @throws Exception
+ * This method may throw exceptions. Throwing an exception will
+ * cause the operation to fail and may trigger recovery.
+ */
+ public IN1 reduce1(IN1 value1, IN1 value2);
+
+ /**
+ * The core method of ReduceFunction, combining two values of the second
+ * input into one value of the same type. The reduce2 function is
+ * consecutively applied to all values of a group until only a single value
+ * remains.
+ *
+ * @param value1
+ * The first value to combine.
+ * @param value2
+ * The second value to combine.
+ * @return The combined value of both input values.
+ *
+ * @throws Exception
+ * This method may throw exceptions. Throwing an exception will
+ * cause the operation to fail and may trigger recovery.
+ */
+ public IN2 reduce2(IN2 value1, IN2 value2);
+
+ /**
+ * Maps the reduced first input to the output type.
+ *
+ * @param value
+ * Type of the first input.
+ * @return the output type.
+ */
+ public OUT map1(IN1 value);
+
+ /**
+ * Maps the reduced second input to the output type.
+ *
+ * @param value
+ * Type of the second input.
+ * @return the output type.
+ */
+ public OUT map2(IN2 value);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoWindowFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoWindowFunction.java
new file mode 100644
index 0000000..2f2514f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoWindowFunction.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.co;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.util.Collector;
+
+public interface CoWindowFunction<IN1, IN2, O> extends Function, Serializable {
+
+ public void coWindow(List<IN1> first, List<IN2> second, Collector<O> out) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java
new file mode 100644
index 0000000..9cafcd1
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.co;
+
+import java.util.List;
+
+import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.util.Collector;
+
+public class CrossWindowFunction<IN1, IN2, OUT> implements CoWindowFunction<IN1, IN2, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ private CrossFunction<IN1, IN2, OUT> crossFunction;
+
+ public CrossWindowFunction(CrossFunction<IN1, IN2, OUT> crossFunction) {
+ this.crossFunction = crossFunction;
+ }
+
+ @Override
+ public void coWindow(List<IN1> first, List<IN2> second, Collector<OUT> out) throws Exception {
+ for (IN1 firstValue : first) {
+ for (IN2 secondValue : second) {
+ out.collect(crossFunction.cross(firstValue, secondValue));
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
new file mode 100644
index 0000000..9b39f33
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.co;
+
+import java.util.List;
+
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.util.Collector;
+
+public class JoinWindowFunction<IN1, IN2, OUT> implements CoWindowFunction<IN1, IN2, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ private KeySelector<IN1, ?> keySelector1;
+ private KeySelector<IN2, ?> keySelector2;
+ private JoinFunction<IN1, IN2, OUT> joinFunction;
+
+ public JoinWindowFunction(KeySelector<IN1, ?> keySelector1, KeySelector<IN2, ?> keySelector2,
+ JoinFunction<IN1, IN2, OUT> joinFunction) {
+ this.keySelector1 = keySelector1;
+ this.keySelector2 = keySelector2;
+ this.joinFunction = joinFunction;
+ }
+
+ @Override
+ public void coWindow(List<IN1> first, List<IN2> second, Collector<OUT> out) throws Exception {
+ for (IN1 item1 : first) {
+ Object key1 = keySelector1.getKey(item1);
+
+ for (IN2 item2 : second) {
+ Object key2 = keySelector2.getKey(item2);
+
+ if (key1.equals(key2)) {
+ out.collect(joinFunction.join(item1, item2));
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoFlatMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoFlatMapFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoFlatMapFunction.java
new file mode 100644
index 0000000..2458f1b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoFlatMapFunction.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.co;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.RichFunction;
+
+/**
+ * A RichCoFlatMapFunction represents a FlatMap transformation with two different input
+ * types. In addition to that the user can use the features provided by the
+ * {@link RichFunction} interface.
+ *
+ * @param <IN1>
+ * Type of the first input.
+ * @param <IN2>
+ * Type of the second input.
+ * @param <OUT>
+ * Output type.
+ */
+public abstract class RichCoFlatMapFunction<IN1, IN2, OUT> extends AbstractRichFunction implements
+ CoFlatMapFunction<IN1, IN2, OUT> {
+
+ private static final long serialVersionUID = 1L;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoMapFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoMapFunction.java
new file mode 100755
index 0000000..20d520c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoMapFunction.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.co;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.RichFunction;
+
+/**
+ * A RichCoMapFunction represents a Map transformation with two different input
+ * types. In addition to that the user can use the features provided by the
+ * {@link RichFunction} interface.
+ *
+ * @param <IN1>
+ * Type of the first input.
+ * @param <IN2>
+ * Type of the second input.
+ * @param <OUT>
+ * Output type.
+ */
+public abstract class RichCoMapFunction<IN1, IN2, OUT> extends AbstractRichFunction implements
+ CoMapFunction<IN1, IN2, OUT> {
+
+ private static final long serialVersionUID = 1L;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoReduceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoReduceFunction.java
new file mode 100644
index 0000000..655923f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoReduceFunction.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.co;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.RichFunction;
+
+/**
+ * A RichCoReduceFunction represents a Reduce transformation with two different input
+ * types. In addition to that the user can use the features provided by the
+ * {@link RichFunction} interface.
+ *
+ * @param <IN1>
+ * Type of the first input.
+ * @param <IN2>
+ * Type of the second input.
+ * @param <OUT>
+ * Output type.
+ */
+public abstract class RichCoReduceFunction<IN1, IN2, OUT> extends AbstractRichFunction implements
+ CoReduceFunction<IN1, IN2, OUT> {
+
+ private static final long serialVersionUID = 1L;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoWindowFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoWindowFunction.java
new file mode 100644
index 0000000..2709203
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoWindowFunction.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.co;
+
+import java.util.List;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.util.Collector;
+
+public abstract class RichCoWindowFunction<IN1, IN2, O> extends AbstractRichFunction implements
+ CoWindowFunction<IN1, IN2, O> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public abstract void coWindow(List<IN1> first, List<IN2> second, Collector<O> out)
+ throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
new file mode 100644
index 0000000..24beba1
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.flink.api.common.io.CleanupWhenUnsuccessful;
+import org.apache.flink.api.common.io.OutputFormat;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Simple implementation of the SinkFunction writing tuples in the specified
+ * OutputFormat format. Tuples are collected to a list and written to the file
+ * periodically. The target path and the overwrite mode are pre-packaged in
+ * format.
+ *
+ * @param <IN>
+ * Input type
+ */
+public abstract class FileSinkFunction<IN> extends RichSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(FileSinkFunction.class);
+ protected ArrayList<IN> tupleList = new ArrayList<IN>();
+ protected volatile OutputFormat<IN> format;
+ protected volatile boolean cleanupCalled = false;
+ protected int indexInSubtaskGroup;
+ protected int currentNumberOfSubtasks;
+
+ public FileSinkFunction(OutputFormat<IN> format) {
+ this.format = format;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
+ format.configure(context.getTaskStubParameters());
+ indexInSubtaskGroup = context.getIndexOfThisSubtask();
+ currentNumberOfSubtasks = context.getNumberOfParallelSubtasks();
+ format.open(indexInSubtaskGroup, currentNumberOfSubtasks);
+ }
+
+ @Override
+ public void invoke(IN record) throws Exception {
+ tupleList.add(record);
+ if (updateCondition()) {
+ flush();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!tupleList.isEmpty()) {
+ flush();
+ }
+ try {
+ format.close();
+ } catch (Exception ex) {
+ try {
+ if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
+ cleanupCalled = true;
+ ((CleanupWhenUnsuccessful) format).tryCleanupOnError();
+ }
+ } catch (Throwable t) {
+ LOG.error("Cleanup on error failed.", t);
+ }
+ }
+ }
+
+ protected void flush() {
+ try {
+ for (IN rec : tupleList) {
+ format.writeRecord(rec);
+ }
+ } catch (Exception ex) {
+ try {
+ if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
+ cleanupCalled = true;
+ ((CleanupWhenUnsuccessful) format).tryCleanupOnError();
+ }
+ } catch (Throwable t) {
+ LOG.error("Cleanup on error failed.", t);
+ }
+ }
+ resetParameters();
+ }
+
+ /**
+ * Condition for writing the contents of tupleList and clearing it.
+ *
+ * @return value of the updating condition
+ */
+ protected abstract boolean updateCondition();
+
+ /**
+ * Statements to be executed after writing a batch goes here.
+ */
+ protected abstract void resetParameters();
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
new file mode 100644
index 0000000..f049a32
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import org.apache.flink.api.common.io.OutputFormat;
+
+/**
+ * Implementation of FileSinkFunction. Writes tuples to file in every millis
+ * milliseconds.
+ *
+ * @param <IN>
+ * Input type
+ */
+public class FileSinkFunctionByMillis<IN> extends FileSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+
+ private final long millis;
+ private long lastTime;
+
+ public FileSinkFunctionByMillis(OutputFormat<IN> format, long millis) {
+ super(format);
+ this.millis = millis;
+ lastTime = System.currentTimeMillis();
+ }
+
+ /**
+ * Condition for writing the contents of tupleList and clearing it.
+ *
+ * @return value of the updating condition
+ */
+ @Override
+ protected boolean updateCondition() {
+ return System.currentTimeMillis() - lastTime >= millis;
+ }
+
+ /**
+ * Statements to be executed after writing a batch goes here.
+ */
+ @Override
+ protected void resetParameters() {
+ tupleList.clear();
+ lastTime = System.currentTimeMillis();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/PrintSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/PrintSinkFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/PrintSinkFunction.java
new file mode 100755
index 0000000..d460749
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/PrintSinkFunction.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import java.io.PrintStream;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
+
+/**
+ * Implementation of the SinkFunction writing every tuple to the standard
+ * output or standard error stream.
+ *
+ * @param <IN>
+ * Input record type
+ */
+public class PrintSinkFunction<IN> extends RichSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+
+ private static final boolean STD_OUT = false;
+ private static final boolean STD_ERR = true;
+
+ private boolean target;
+ private transient PrintStream stream;
+ private transient String prefix;
+
+ /**
+ * Instantiates a print sink function that prints to standard out.
+ */
+ public PrintSinkFunction() {}
+
+ /**
+ * Instantiates a print sink function that prints to standard out.
+ *
+ * @param stdErr True, if the format should print to standard error instead of standard out.
+ */
+ public PrintSinkFunction(boolean stdErr) {
+ target = stdErr;
+ }
+
+ public void setTargetToStandardOut() {
+ target = STD_OUT;
+ }
+
+ public void setTargetToStandardErr() {
+ target = STD_ERR;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
+ // get the target stream
+ stream = target == STD_OUT ? System.out : System.err;
+
+ // set the prefix if we have a >1 DOP
+ prefix = (context.getNumberOfParallelSubtasks() > 1) ?
+ ((context.getIndexOfThisSubtask() + 1) + "> ") : null;
+ }
+
+ @Override
+ public void invoke(IN record) {
+ if (prefix != null) {
+ stream.println(prefix + record.toString());
+ }
+ else {
+ stream.println(record.toString());
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.stream = null;
+ this.prefix = null;
+ super.close();
+ }
+
+ @Override
+ public String toString() {
+ return "Print to " + (target == STD_OUT ? "System.out" : "System.err");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
new file mode 100755
index 0000000..3b8a4db
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+
+public abstract class RichSinkFunction<IN> extends AbstractRichFunction implements SinkFunction<IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ public abstract void invoke(IN value) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
new file mode 100644
index 0000000..6097603
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.Function;
+
+public interface SinkFunction<IN> extends Function, Serializable {
+
+ public abstract void invoke(IN value) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormat.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormat.java
new file mode 100644
index 0000000..a606302
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormat.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+/**
+ * Abstract class for formatting the output of the writeAsText and writeAsCsv
+ * functions.
+ *
+ * @param <IN>
+ * Input tuple type
+ */
+public abstract class WriteFormat<IN> implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Writes the contents of tupleList to the file specified by path.
+ *
+ * @param path
+ * is the path to the location where the tuples are written
+ * @param tupleList
+ * is the list of tuples to be written
+ */
+ protected abstract void write(String path, ArrayList<IN> tupleList);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java
new file mode 100644
index 0000000..b22fd80
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+
+/**
+ * Writes tuples in csv format.
+ *
+ * @param <IN>
+ * Input tuple type
+ */
+public class WriteFormatAsCsv<IN> extends WriteFormat<IN> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected void write(String path, ArrayList<IN> tupleList) {
+ try {
+ PrintWriter outStream = new PrintWriter(new BufferedWriter(new FileWriter(path, true)));
+ for (IN tupleToWrite : tupleList) {
+ String strTuple = tupleToWrite.toString();
+ outStream.println(strTuple.substring(1, strTuple.length() - 1));
+ }
+ outStream.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Exception occured while writing file " + path, e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsText.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsText.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsText.java
new file mode 100644
index 0000000..5891104
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsText.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+
+/**
+ * Writes tuples in text format.
+ *
+ * @param <IN>
+ * Input tuple type
+ */
+public class WriteFormatAsText<IN> extends WriteFormat<IN> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void write(String path, ArrayList<IN> tupleList) {
+ try {
+ PrintWriter outStream = new PrintWriter(new BufferedWriter(new FileWriter(path, true)));
+ for (IN tupleToWrite : tupleList) {
+ outStream.println(tupleToWrite);
+ }
+ outStream.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Exception occured while writing file " + path, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java
new file mode 100644
index 0000000..0c52afc
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+
+/**
+ * Simple implementation of the SinkFunction writing tuples as simple text to
+ * the file specified by path. Tuples are collected to a list and written to the
+ * file periodically. The file specified by path is created if it does not
+ * exist, cleared if it exists before the writing.
+ *
+ * @param <IN>
+ * Input tuple type
+ */
+public abstract class WriteSinkFunction<IN> implements SinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+
+ protected final String path;
+ protected ArrayList<IN> tupleList = new ArrayList<IN>();
+ protected WriteFormat<IN> format;
+
+ public WriteSinkFunction(String path, WriteFormat<IN> format) {
+ this.path = path;
+ this.format = format;
+ cleanFile(path);
+ }
+
+ /**
+ * Creates target file if it does not exist, cleans it if it exists.
+ *
+ * @param path
+ * is the path to the location where the tuples are written
+ */
+ protected void cleanFile(String path) {
+ try {
+ PrintWriter writer;
+ writer = new PrintWriter(path);
+ writer.print("");
+ writer.close();
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException("File not found " + path, e);
+ }
+ }
+
+ /**
+ * Condition for writing the contents of tupleList and clearing it.
+ *
+ * @return value of the updating condition
+ */
+ protected abstract boolean updateCondition();
+
+ /**
+ * Statements to be executed after writing a batch goes here.
+ */
+ protected abstract void resetParameters();
+
+ /**
+ * Implementation of the invoke method of the SinkFunction class. Collects
+ * the incoming tuples in tupleList and appends the list to the end of the
+ * target file if updateCondition() is true or the current tuple is the
+ * endTuple.
+ */
+ @Override
+ public void invoke(IN tuple) {
+
+ tupleList.add(tuple);
+ if (updateCondition()) {
+ format.write(path, tupleList);
+ resetParameters();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java
new file mode 100644
index 0000000..ee6df94
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+/**
+ * Implementation of WriteSinkFunction. Writes tuples to file in every millis
+ * milliseconds.
+ *
+ * @param <IN>
+ * Input tuple type
+ */
+public class WriteSinkFunctionByMillis<IN> extends WriteSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+
+ private final long millis;
+ private long lastTime;
+
+ public WriteSinkFunctionByMillis(String path, WriteFormat<IN> format, long millis) {
+ super(path, format);
+ this.millis = millis;
+ lastTime = System.currentTimeMillis();
+ }
+
+ @Override
+ protected boolean updateCondition() {
+ return System.currentTimeMillis() - lastTime >= millis;
+ }
+
+ @Override
+ protected void resetParameters() {
+ tupleList.clear();
+ lastTime = System.currentTimeMillis();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
new file mode 100644
index 0000000..05a2489
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileMonitoringFunction implements SourceFunction<Tuple3<String, Long, Long>> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(FileMonitoringFunction.class);
+
+ public enum WatchType {
+ ONLY_NEW_FILES, // Only new files will be processed.
+ REPROCESS_WITH_APPENDED, // When some files are appended, all contents of the files will be processed.
+ PROCESS_ONLY_APPENDED // When some files are appended, only appended contents will be processed.
+ }
+
+ private String path;
+ private long interval;
+ private WatchType watchType;
+
+ private FileSystem fileSystem;
+ private Map<String, Long> offsetOfFiles;
+ private Map<String, Long> modificationTimes;
+
+ public FileMonitoringFunction(String path, long interval, WatchType watchType) {
+ this.path = path;
+ this.interval = interval;
+ this.watchType = watchType;
+ this.modificationTimes = new HashMap<String, Long>();
+ this.offsetOfFiles = new HashMap<String, Long>();
+ }
+
+ @Override
+ public void invoke(Collector<Tuple3<String, Long, Long>> collector) throws Exception {
+ fileSystem = FileSystem.get(new URI(path));
+
+ while (true) {
+ List<String> files = listNewFiles();
+ for (String filePath : files) {
+ if (watchType == WatchType.ONLY_NEW_FILES
+ || watchType == WatchType.REPROCESS_WITH_APPENDED) {
+ collector.collect(new Tuple3<String, Long, Long>(filePath, 0L, -1L));
+ offsetOfFiles.put(filePath, -1L);
+ } else if (watchType == WatchType.PROCESS_ONLY_APPENDED) {
+ long offset = 0;
+ long fileSize = fileSystem.getFileStatus(new Path(filePath)).getLen();
+ if (offsetOfFiles.containsKey(filePath)) {
+ offset = offsetOfFiles.get(filePath);
+ }
+
+ collector.collect(new Tuple3<String, Long, Long>(filePath, offset, fileSize));
+ offsetOfFiles.put(filePath, fileSize);
+
+ LOG.info("File processed: {}, {}, {}", filePath, offset, fileSize);
+ }
+ }
+
+ Thread.sleep(interval);
+ }
+ }
+
+ private List<String> listNewFiles() throws IOException {
+ List<String> files = new ArrayList<String>();
+
+ FileStatus[] statuses = fileSystem.listStatus(new Path(path));
+
+ for (FileStatus status : statuses) {
+ Path filePath = status.getPath();
+ String fileName = filePath.getName();
+ long modificationTime = status.getModificationTime();
+
+ if (!isFiltered(fileName, modificationTime)) {
+ files.add(filePath.toString());
+ modificationTimes.put(fileName, modificationTime);
+ }
+ }
+ return files;
+ }
+
+ private boolean isFiltered(String fileName, long modificationTime) {
+
+ if ((watchType == WatchType.ONLY_NEW_FILES && modificationTimes.containsKey(fileName))
+ || fileName.startsWith(".") || fileName.contains("_COPYING_")) {
+ return true;
+ } else {
+ Long lastModification = modificationTimes.get(fileName);
+ if (lastModification == null) {
+ return false;
+ } else {
+ return lastModification >= modificationTime;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java
new file mode 100644
index 0000000..0882d9e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.util.Collector;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.URI;
+
+public class FileReadFunction implements FlatMapFunction<Tuple3<String, Long, Long>, String> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception {
+ FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0));
+ stream.seek(value.f1);
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
+ String line;
+
+ try {
+ while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) {
+ out.collect(line);
+ }
+ } finally {
+ reader.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java
new file mode 100644
index 0000000..5dfe4b2
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.api.java.typeutils.runtime.RuntimeStatefulSerializerFactory;
+import org.apache.flink.api.java.typeutils.runtime.RuntimeStatelessSerializerFactory;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.io.InputSplit;
+import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
+import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
+import org.apache.flink.util.Collector;
+
+public class FileSourceFunction extends RichSourceFunction<String> {
+ private static final long serialVersionUID = 1L;
+
+ private InputSplitProvider provider;
+
+ private InputFormat<String, ?> inputFormat;
+
+ private TypeSerializerFactory<String> serializerFactory;
+
+ public FileSourceFunction(InputFormat<String, ?> format, TypeInformation<String> typeInfo) {
+ this.inputFormat = format;
+ this.serializerFactory = createSerializer(typeInfo);
+ }
+
+ private static TypeSerializerFactory<String> createSerializer(TypeInformation<String> typeInfo) {
+ TypeSerializer<String> serializer = typeInfo.createSerializer();
+
+ if (serializer.isStateful()) {
+ return new RuntimeStatefulSerializerFactory<String>(serializer, typeInfo.getTypeClass());
+ } else {
+ return new RuntimeStatelessSerializerFactory<String>(serializer,
+ typeInfo.getTypeClass());
+ }
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
+ this.provider = context.getInputSplitProvider();
+ inputFormat.configure(context.getTaskStubParameters());
+ }
+
+ @Override
+ public void invoke(Collector<String> collector) throws Exception {
+ final TypeSerializer<String> serializer = serializerFactory.getSerializer();
+ final Iterator<InputSplit> splitIterator = getInputSplits();
+ @SuppressWarnings("unchecked")
+ final InputFormat<String, InputSplit> format = (InputFormat<String, InputSplit>) this.inputFormat;
+ try {
+ while (splitIterator.hasNext()) {
+
+ final InputSplit split = splitIterator.next();
+ String record = serializer.createInstance();
+
+ format.open(split);
+ try {
+ while (!format.reachedEnd()) {
+ if ((record = format.nextRecord(record)) != null) {
+ collector.collect(record);
+ }
+ }
+ } finally {
+ format.close();
+ }
+ }
+ collector.close();
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ private Iterator<InputSplit> getInputSplits() {
+
+ return new Iterator<InputSplit>() {
+
+ private InputSplit nextSplit;
+
+ private boolean exhausted;
+
+ @Override
+ public boolean hasNext() {
+ if (exhausted) {
+ return false;
+ }
+
+ if (nextSplit != null) {
+ return true;
+ }
+
+ InputSplit split = provider.getNextInputSplit();
+
+ if (split != null) {
+ this.nextSplit = split;
+ return true;
+ } else {
+ exhausted = true;
+ return false;
+ }
+ }
+
+ @Override
+ public InputSplit next() {
+ if (this.nextSplit == null && !hasNext()) {
+ throw new NoSuchElementException();
+ }
+
+ final InputSplit tmp = this.nextSplit;
+ this.nextSplit = null;
+ return tmp;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+}
[12/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java
new file mode 100755
index 0000000..8afac75
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.flink.util.Collector;
+
+public class FromElementsFunction<T> implements SourceFunction<T> {
+ private static final long serialVersionUID = 1L;
+
+ Iterable<T> iterable;
+
+ public FromElementsFunction(T... elements) {
+ this.iterable = Arrays.asList(elements);
+ }
+
+ public FromElementsFunction(Collection<T> elements) {
+ this.iterable = elements;
+ }
+
+ public FromElementsFunction(Iterable<T> elements) {
+ this.iterable = elements;
+ }
+
+ @Override
+ public void invoke(Collector<T> collector) throws Exception {
+ for (T element : iterable) {
+ collector.collect(element);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java
new file mode 100755
index 0000000..3afd06e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.NumberSequenceIterator;
+
+/**
+ * Source Function used to generate the number sequence
+ *
+ */
+public class GenSequenceFunction extends RichParallelSourceFunction<Long> {
+
+ private static final long serialVersionUID = 1L;
+
+ private NumberSequenceIterator fullIterator;
+ private NumberSequenceIterator splitIterator;
+
+ public GenSequenceFunction(long from, long to) {
+ fullIterator = new NumberSequenceIterator(from, to);
+ }
+
+ @Override
+ public void invoke(Collector<Long> collector) throws Exception {
+ while (splitIterator.hasNext()) {
+ collector.collect(splitIterator.next());
+ }
+ }
+
+ @Override
+ public void open(Configuration config) {
+ int splitNumber = getRuntimeContext().getIndexOfThisSubtask();
+ int numOfSubTasks = getRuntimeContext().getNumberOfParallelSubtasks();
+ splitIterator = fullIterator.split(numOfSubTasks)[splitNumber];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java
new file mode 100644
index 0000000..664d39a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+
+public interface GenericSourceFunction<T> {
+
+ public TypeInformation<T> getType();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
new file mode 100644
index 0000000..46d4fe9
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.util.Collector;
+
+public interface ParallelSourceFunction<OUT> extends SourceFunction<OUT> {
+
+ public void invoke(Collector<OUT> collector) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java
new file mode 100644
index 0000000..5bbfd4c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+
+public abstract class RichParallelSourceFunction<OUT> extends AbstractRichFunction implements
+ ParallelSourceFunction<OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java
new file mode 100755
index 0000000..4b947c7
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+
+public abstract class RichSourceFunction<OUT> extends AbstractRichFunction implements
+ SourceFunction<OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SocketTextStreamFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SocketTextStreamFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SocketTextStreamFunction.java
new file mode 100644
index 0000000..ac82b10
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SocketTextStreamFunction.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.Collector;
+
+public class SocketTextStreamFunction extends RichSourceFunction<String> {
+ private static final long serialVersionUID = 1L;
+
+ private String hostname;
+ private int port;
+ private char delimiter;
+ private Socket socket;
+ private static final int CONNECTION_TIMEOUT_TIME = 0;
+
+ public SocketTextStreamFunction(String hostname, int port, char delimiter) {
+ this.hostname = hostname;
+ this.port = port;
+ this.delimiter = delimiter;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ socket = new Socket();
+
+ socket.connect(new InetSocketAddress(hostname, port), CONNECTION_TIMEOUT_TIME);
+ }
+
+ @Override
+ public void invoke(Collector<String> collector) throws Exception {
+ while (!socket.isClosed() && socket.isConnected()) {
+ streamFromSocket(collector, socket);
+ }
+ }
+
+ public void streamFromSocket(Collector<String> collector, Socket socket) throws Exception {
+ StringBuffer buffer = new StringBuffer();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream()));
+
+ while (true) {
+ int data = reader.read();
+ if (!socket.isConnected() || socket.isClosed() || data == -1) {
+ break;
+ }
+
+ if (data == delimiter) {
+ collector.collect(buffer.toString());
+ buffer = new StringBuffer();
+ } else if (data != '\r') { // ignore carriage return
+ buffer.append((char) data);
+ }
+ }
+
+ if (buffer.length() > 0) {
+ collector.collect(buffer.toString());
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ socket.close();
+ super.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SourceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SourceFunction.java
new file mode 100755
index 0000000..917562a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SourceFunction.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.util.Collector;
+
+public interface SourceFunction<OUT> extends Function, Serializable {
+
+ public void invoke(Collector<OUT> collector) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
new file mode 100644
index 0000000..24c0319
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.util.Collector;
+
+public abstract class ChainableInvokable<IN, OUT> extends StreamInvokable<IN, OUT> implements
+ Collector<IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ public ChainableInvokable(Function userFunction) {
+ super(userFunction);
+ setChainingStrategy(ChainingStrategy.ALWAYS);
+ }
+
+ public void setup(Collector<OUT> collector, StreamRecordSerializer<IN> inSerializer) {
+ this.collector = collector;
+ this.inSerializer = inSerializer;
+ this.objectSerializer = inSerializer.getObjectSerializer();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
new file mode 100755
index 0000000..13a6ba1
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable;
+
+import org.apache.flink.streaming.api.function.sink.SinkFunction;
+
+public class SinkInvokable<IN> extends ChainableInvokable<IN, IN> {
+ private static final long serialVersionUID = 1L;
+
+ private SinkFunction<IN> sinkFunction;
+
+ public SinkInvokable(SinkFunction<IN> sinkFunction) {
+ super(sinkFunction);
+ this.sinkFunction = sinkFunction;
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ while (readNext() != null) {
+ callUserFunctionAndLogException();
+ }
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ sinkFunction.invoke(nextObject);
+ }
+
+ @Override
+ public void collect(IN record) {
+ nextObject = copy(record);
+ callUserFunctionAndLogException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SourceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SourceInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SourceInvokable.java
new file mode 100644
index 0000000..f1cf2c5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SourceInvokable.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable;
+
+import java.io.Serializable;
+
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+
+public class SourceInvokable<OUT> extends StreamInvokable<OUT, OUT> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private SourceFunction<OUT> sourceFunction;
+
+ public SourceInvokable(SourceFunction<OUT> sourceFunction) {
+ super(sourceFunction);
+ this.sourceFunction = sourceFunction;
+ }
+
+ @Override
+ public void invoke() {
+ callUserFunctionAndLogException();
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ sourceFunction.invoke(collector);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
new file mode 100644
index 0000000..6cee5f2
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.api.streamvertex.StreamTaskContext;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.MutableObjectIterator;
+import org.apache.flink.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The StreamInvokable represents the base class for all invokables in the
+ * streaming topology.
+ *
+ * @param <OUT>
+ * The output type of the invokable
+ */
+public abstract class StreamInvokable<IN, OUT> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(StreamInvokable.class);
+
+ protected StreamTaskContext<OUT> taskContext;
+
+ protected MutableObjectIterator<StreamRecord<IN>> recordIterator;
+ protected StreamRecordSerializer<IN> inSerializer;
+ protected TypeSerializer<IN> objectSerializer;
+ protected StreamRecord<IN> nextRecord;
+ protected IN nextObject;
+ protected boolean isMutable;
+
+ protected Collector<OUT> collector;
+ protected Function userFunction;
+ protected volatile boolean isRunning;
+
+ private ChainingStrategy chainingStrategy = ChainingStrategy.HEAD;
+
+ public StreamInvokable(Function userFunction) {
+ this.userFunction = userFunction;
+ }
+
+ /**
+ * Initializes the {@link StreamInvokable} for input and output handling
+ *
+ * @param taskContext
+ * StreamTaskContext representing the vertex
+ */
+ public void setup(StreamTaskContext<OUT> taskContext) {
+ this.collector = taskContext.getOutputCollector();
+ this.recordIterator = taskContext.getInput(0);
+ this.inSerializer = taskContext.getInputSerializer(0);
+ if (this.inSerializer != null) {
+ this.nextRecord = inSerializer.createInstance();
+ this.objectSerializer = inSerializer.getObjectSerializer();
+ }
+ this.taskContext = taskContext;
+ }
+
+ /**
+ * Method that will be called when the operator starts, should encode the
+ * processing logic
+ */
+ public abstract void invoke() throws Exception;
+
+ /*
+ * Reads the next record from the reader iterator and stores it in the
+ * nextRecord variable
+ */
+ protected StreamRecord<IN> readNext() {
+ this.nextRecord = inSerializer.createInstance();
+ try {
+ nextRecord = recordIterator.next(nextRecord);
+ try {
+ nextObject = nextRecord.getObject();
+ } catch (NullPointerException e) {
+ // end of stream
+ }
+ return nextRecord;
+ } catch (IOException e) {
+ throw new RuntimeException("Could not read next record.");
+ }
+ }
+
+ /**
+ * The call of the user implemented function should be implemented here
+ */
+ protected void callUserFunction() throws Exception {
+ }
+
+ /**
+ * Method for logging exceptions thrown during the user function call
+ */
+ protected void callUserFunctionAndLogException() {
+ try {
+ callUserFunction();
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Calling user function failed due to: {}",
+ StringUtils.stringifyException(e));
+ }
+ }
+ }
+
+ /**
+ * Open method to be used if the user defined function extends the
+ * RichFunction class
+ *
+ * @param parameters
+ * The configuration parameters for the operator
+ */
+ public void open(Configuration parameters) throws Exception {
+ isRunning = true;
+ FunctionUtils.openFunction(userFunction, parameters);
+ }
+
+ /**
+ * Close method to be used if the user defined function extends the
+ * RichFunction class
+ *
+ */
+ public void close() {
+ isRunning = false;
+ collector.close();
+ try {
+ FunctionUtils.closeFunction(userFunction);
+ } catch (Exception e) {
+ throw new RuntimeException("Error when closing the function: " + e.getMessage());
+ }
+ }
+
+ public void setRuntimeContext(RuntimeContext t) {
+ FunctionUtils.setFunctionRuntimeContext(userFunction, t);
+ }
+
+ protected IN copy(IN record) {
+ return objectSerializer.copy(record);
+ }
+
+ public void setChainingStrategy(ChainingStrategy strategy) {
+ if (strategy == ChainingStrategy.ALWAYS) {
+ if (!(this instanceof ChainableInvokable)) {
+ throw new RuntimeException(
+ "Invokable needs to extend ChainableInvokable to be chained");
+ }
+ }
+ this.chainingStrategy = strategy;
+ }
+
+ public ChainingStrategy getChainingStrategy() {
+ return chainingStrategy;
+ }
+
+ public static enum ChainingStrategy {
+ ALWAYS, NEVER, HEAD;
+ }
+
+ public Function getUserFunction() {
+ return userFunction;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchIterator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchIterator.java
new file mode 100755
index 0000000..c3475e9
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchIterator.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+public interface BatchIterator<IN> extends Iterator<IN>, Serializable {
+ public void reset();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
new file mode 100644
index 0000000..3fc314c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+
+public class CounterInvokable<IN> extends ChainableInvokable<IN, Long> {
+ private static final long serialVersionUID = 1L;
+
+ Long count = 0L;
+
+ public CounterInvokable() {
+ super(null);
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ while (readNext() != null) {
+ collector.collect(++count);
+ }
+ }
+
+ @Override
+ public void collect(IN record) {
+ collector.collect(++count);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
new file mode 100644
index 0000000..0c8298e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+
+public class FilterInvokable<IN> extends ChainableInvokable<IN, IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ FilterFunction<IN> filterFunction;
+ private boolean collect;
+
+ public FilterInvokable(FilterFunction<IN> filterFunction) {
+ super(filterFunction);
+ this.filterFunction = filterFunction;
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ while (readNext() != null) {
+ callUserFunctionAndLogException();
+ }
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ collect = filterFunction.filter(copy(nextObject));
+ if (collect) {
+ collector.collect(nextObject);
+ }
+ }
+
+ @Override
+ public void collect(IN record) {
+ nextObject = copy(record);
+ callUserFunctionAndLogException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
new file mode 100644
index 0000000..2a4081b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+
+public class FlatMapInvokable<IN, OUT> extends ChainableInvokable<IN, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ private FlatMapFunction<IN, OUT> flatMapper;
+
+ public FlatMapInvokable(FlatMapFunction<IN, OUT> flatMapper) {
+ super(flatMapper);
+ this.flatMapper = flatMapper;
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ while (readNext() != null) {
+ callUserFunctionAndLogException();
+ }
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ flatMapper.flatMap(nextObject, collector);
+ }
+
+ @Override
+ public void collect(IN record) {
+ nextObject = copy(record);
+ callUserFunctionAndLogException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
new file mode 100755
index 0000000..c2177fa
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+
+public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
+ private static final long serialVersionUID = 1L;
+
+ private KeySelector<IN, ?> keySelector;
+ private Map<Object, IN> values;
+ private IN reduced;
+
+ public GroupedReduceInvokable(ReduceFunction<IN> reducer, KeySelector<IN, ?> keySelector) {
+ super(reducer);
+ this.keySelector = keySelector;
+ values = new HashMap<Object, IN>();
+ }
+
+ @Override
+ protected void reduce() throws Exception {
+ Object key = nextRecord.getKey(keySelector);
+ currentValue = values.get(key);
+ nextValue = nextObject;
+ if (currentValue != null) {
+ callUserFunctionAndLogException();
+ values.put(key, reduced);
+ collector.collect(reduced);
+ } else {
+ values.put(key, nextValue);
+ collector.collect(nextValue);
+ }
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ reduced = reducer.reduce(currentValue, nextValue);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
new file mode 100644
index 0000000..997463c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
@@ -0,0 +1,506 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+/**
+ * This invokable allows windowing based on {@link TriggerPolicy} and
+ * {@link EvictionPolicy} instances including their active and cloneable
+ * versions. It is additionally aware of the creation of windows per group.
+ *
+ * A {@link KeySelector} is used to specify the key position or key extraction.
+ * The {@link ReduceFunction} will be executed on each group separately.
+ * Policies might either be centralized or distributed. It is not possible to
+ * use central and distributed eviction policies at the same time. A distributed
+ * policy have to be a {@link CloneableTriggerPolicy} or
+ * {@link CloneableEvictionPolicy} as it will be cloned to have separated
+ * instances for each group. At the startup time the distributed policies will
+ * be stored as sample, and only clones of them will be used to maintain the
+ * groups. Therefore, each group starts with the initial policy states.
+ *
+ * While a distributed policy only gets notified with the elements belonging to
+ * the respective group, a centralized policy get notified with all arriving
+ * elements. When a centralized trigger occurred, all groups get triggered. This
+ * is done by submitting the element which caused the trigger as real element to
+ * the groups it belongs to and as fake element to all other groups. Within the
+ * groups the element might be further processed, causing more triggers,
+ * prenotifications of active distributed policies and evictions like usual.
+ *
+ * Central policies can be instance of {@link ActiveTriggerPolicy} and also
+ * implement the
+ * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
+ * method. Fake elements created on prenotification will be forwarded to all
+ * groups. The {@link ActiveTriggerCallback} is also implemented in a way, that
+ * it forwards/distributed calls all groups.
+ *
+ * @param <IN>
+ * The type of input elements handled by this operator invokable.
+ */
+public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
+
+ /**
+ * Auto-generated serial version UID
+ */
+ private static final long serialVersionUID = -3469545957144404137L;
+
+ private KeySelector<IN, ?> keySelector;
+ private Configuration parameters;
+ private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies;
+ private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies;
+ private LinkedList<ActiveEvictionPolicy<IN>> activeCentralEvictionPolicies;
+ private LinkedList<EvictionPolicy<IN>> centralEvictionPolicies;
+ private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies;
+ private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies;
+ private Map<Object, WindowInvokable<IN, OUT>> windowingGroups;
+ private LinkedList<Thread> activePolicyThreads;
+ private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
+ private LinkedList<WindowInvokable<IN, OUT>> deleteOrderForCentralEviction;
+
+ /**
+ * This constructor creates an instance of the grouped windowing invokable.
+ *
+ * A {@link KeySelector} is used to specify the key position or key
+ * extraction. The {@link ReduceFunction} will be executed on each group
+ * separately. Policies might either be centralized or distributed. It is
+ * not possible to use central and distributed eviction policies at the same
+ * time. A distributed policy have to be a {@link CloneableTriggerPolicy} or
+ * {@link CloneableEvictionPolicy} as it will be cloned to have separated
+ * instances for each group. At the startup time the distributed policies
+ * will be stored as sample, and only clones of them will be used to
+ * maintain the groups. Therefore, each group starts with the initial policy
+ * states.
+ *
+ * While a distributed policy only gets notified with the elements belonging
+ * to the respective group, a centralized policy get notified with all
+ * arriving elements. When a centralized trigger occurred, all groups get
+ * triggered. This is done by submitting the element which caused the
+ * trigger as real element to the groups it belongs to and as fake element
+ * to all other groups. Within the groups the element might be further
+ * processed, causing more triggers, prenotifications of active distributed
+ * policies and evictions like usual.
+ *
+ * Central policies can be instance of {@link ActiveTriggerPolicy} and also
+ * implement the
+ * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
+ * method. Fake elements created on prenotification will be forwarded to all
+ * groups. The {@link ActiveTriggerCallback} is also implemented in a way,
+ * that it forwards/distributed calls all groups.
+ *
+ * @param userFunction
+ * The user defined function.
+ * @param keySelector
+ * A key selector to extract the key for the groups from the
+ * input data.
+ * @param distributedTriggerPolicies
+ * Trigger policies to be distributed and maintained individually
+ * within each group.
+ * @param distributedEvictionPolicies
+ * Eviction policies to be distributed and maintained
+ * individually within each group. Note that there cannot be
+ * both, central and distributed eviction policies at the same
+ * time.
+ * @param centralTriggerPolicies
+ * Trigger policies which will only exist once at a central
+ * place. In case a central policy triggers, it will cause all
+ * groups to be emitted. (Remark: Empty groups cannot be emitted.
+ * If only one element is contained a group, this element itself
+ * is returned as aggregated result.)
+ * @param centralEvictionPolicies
+ * Eviction which will only exist once at a central place. Note
+ * that there cannot be both, central and distributed eviction
+ * policies at the same time. The central eviction policy will
+ * work on an simulated element buffer containing all elements no
+ * matter which group they belong to.
+ */
+ public GroupedWindowInvokable(Function userFunction, KeySelector<IN, ?> keySelector,
+ LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
+ LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
+ LinkedList<TriggerPolicy<IN>> centralTriggerPolicies,
+ LinkedList<EvictionPolicy<IN>> centralEvictionPolicies) {
+
+ super(userFunction);
+
+ this.keySelector = keySelector;
+
+ // handle the triggers
+ if (centralTriggerPolicies != null) {
+ this.centralTriggerPolicies = centralTriggerPolicies;
+ this.activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
+
+ for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
+ if (trigger instanceof ActiveTriggerPolicy) {
+ this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
+ }
+ }
+ } else {
+ this.centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ }
+
+ if (distributedTriggerPolicies != null) {
+ this.distributedTriggerPolicies = distributedTriggerPolicies;
+ } else {
+ this.distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
+ }
+
+ if (distributedEvictionPolicies != null) {
+ this.distributedEvictionPolicies = distributedEvictionPolicies;
+ } else {
+ this.distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
+ }
+
+ this.activeCentralEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
+
+ if (centralEvictionPolicies != null) {
+ this.centralEvictionPolicies = centralEvictionPolicies;
+
+ for (EvictionPolicy<IN> eviction : centralEvictionPolicies) {
+ if (eviction instanceof ActiveEvictionPolicy) {
+ this.activeCentralEvictionPolicies.add((ActiveEvictionPolicy<IN>) eviction);
+ }
+ }
+ } else {
+ this.centralEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
+ }
+
+ this.windowingGroups = new HashMap<Object, WindowInvokable<IN, OUT>>();
+ this.activePolicyThreads = new LinkedList<Thread>();
+ this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ this.deleteOrderForCentralEviction = new LinkedList<WindowInvokable<IN, OUT>>();
+
+ // check that not both, central and distributed eviction, is used at the
+ // same time.
+ if (!this.centralEvictionPolicies.isEmpty() && !this.distributedEvictionPolicies.isEmpty()) {
+ throw new UnsupportedOperationException(
+ "You can only use either central or distributed eviction policies but not both at the same time.");
+ }
+
+ // Check that there is at least one trigger and one eviction policy
+ if (this.centralEvictionPolicies.isEmpty() && this.distributedEvictionPolicies.isEmpty()) {
+ throw new UnsupportedOperationException(
+ "You have to define at least one eviction policy");
+ }
+ if (this.centralTriggerPolicies.isEmpty() && this.distributedTriggerPolicies.isEmpty()) {
+ throw new UnsupportedOperationException(
+ "You have to define at least one trigger policy");
+ }
+
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ // Prevent empty data streams
+ if (readNext() == null) {
+ throw new RuntimeException("DataStream must not be empty");
+ }
+
+ // Continuously run
+ while (nextRecord != null) {
+ WindowInvokable<IN, OUT> groupInvokable = windowingGroups.get(keySelector
+ .getKey(nextRecord.getObject()));
+ if (groupInvokable == null) {
+ groupInvokable = makeNewGroup(nextRecord);
+ }
+
+ // Run the precalls for central active triggers
+ for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
+ Object[] result = trigger.preNotifyTrigger(nextRecord.getObject());
+ for (Object in : result) {
+
+ // If central eviction is used, handle it here
+ if (!activeCentralEvictionPolicies.isEmpty()) {
+ evictElements(centralActiveEviction(in));
+ }
+
+ // process in groups
+ for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
+ group.processFakeElement(in, trigger);
+ checkForEmptyGroupBuffer(group);
+ }
+ }
+ }
+
+ // Process non-active central triggers
+ for (TriggerPolicy<IN> triggerPolicy : centralTriggerPolicies) {
+ if (triggerPolicy.notifyTrigger(nextRecord.getObject())) {
+ currentTriggerPolicies.add(triggerPolicy);
+ }
+ }
+
+ if (currentTriggerPolicies.isEmpty()) {
+
+ // only add the element to its group
+ groupInvokable.processRealElement(nextRecord.getObject());
+ checkForEmptyGroupBuffer(groupInvokable);
+
+ // If central eviction is used, handle it here
+ if (!centralEvictionPolicies.isEmpty()) {
+ evictElements(centralEviction(nextRecord.getObject(), false));
+ deleteOrderForCentralEviction.add(groupInvokable);
+ }
+
+ } else {
+
+ // call user function for all groups
+ for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
+ if (group == groupInvokable) {
+ // process real with initialized policies
+ group.processRealElement(nextRecord.getObject(), currentTriggerPolicies);
+ } else {
+ // process like a fake but also initialized with
+ // policies
+ group.externalTriggerFakeElement(nextRecord.getObject(),
+ currentTriggerPolicies);
+ }
+
+ // remove group in case it has an empty buffer
+ // checkForEmptyGroupBuffer(group);
+ }
+
+ // If central eviction is used, handle it here
+ if (!centralEvictionPolicies.isEmpty()) {
+ evictElements(centralEviction(nextRecord.getObject(), true));
+ deleteOrderForCentralEviction.add(groupInvokable);
+ }
+ }
+
+ // clear current trigger list
+ currentTriggerPolicies.clear();
+
+ // read next record
+ readNext();
+ }
+
+ // Stop all remaining threads from policies
+ for (Thread t : activePolicyThreads) {
+ t.interrupt();
+ }
+
+ // finally trigger the buffer.
+ for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
+ group.emitFinalWindow(centralTriggerPolicies);
+ }
+
+ }
+
+ /**
+ * This method creates a new group. The method gets called in case an
+ * element arrives which has a key which was not seen before. The method
+ * created a nested {@link WindowInvokable} and therefore created clones of
+ * all distributed trigger and eviction policies.
+ *
+ * @param element
+ * The element which leads to the generation of a new group
+ * (previously unseen key)
+ * @throws Exception
+ * In case the {@link KeySelector} throws an exception in
+ * {@link KeySelector#getKey(Object)}, the exception is not
+ * catched by this method.
+ */
+ @SuppressWarnings("unchecked")
+ private WindowInvokable<IN, OUT> makeNewGroup(StreamRecord<IN> element) throws Exception {
+ // clone the policies
+ LinkedList<TriggerPolicy<IN>> clonedDistributedTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ LinkedList<EvictionPolicy<IN>> clonedDistributedEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
+ for (CloneableTriggerPolicy<IN> trigger : this.distributedTriggerPolicies) {
+ clonedDistributedTriggerPolicies.add(trigger.clone());
+ }
+ for (CloneableEvictionPolicy<IN> eviction : this.distributedEvictionPolicies) {
+ clonedDistributedEvictionPolicies.add(eviction.clone());
+ }
+
+ WindowInvokable<IN, OUT> groupInvokable;
+ if (userFunction instanceof ReduceFunction) {
+ groupInvokable = (WindowInvokable<IN, OUT>) new WindowReduceInvokable<IN>(
+ (ReduceFunction<IN>) userFunction, clonedDistributedTriggerPolicies,
+ clonedDistributedEvictionPolicies);
+ } else {
+ groupInvokable = new WindowGroupReduceInvokable<IN, OUT>(
+ (GroupReduceFunction<IN, OUT>) userFunction, clonedDistributedTriggerPolicies,
+ clonedDistributedEvictionPolicies);
+ }
+
+ groupInvokable.setup(taskContext);
+ groupInvokable.open(this.parameters);
+ windowingGroups.put(keySelector.getKey(element.getObject()), groupInvokable);
+
+ return groupInvokable;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ this.parameters = parameters;
+ for (ActiveTriggerPolicy<IN> tp : activeCentralTriggerPolicies) {
+ Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
+ if (target != null) {
+ Thread thread = new Thread(target);
+ activePolicyThreads.add(thread);
+ thread.start();
+ }
+ }
+ };
+
+ /**
+ * This method is used to notify central eviction policies with a real
+ * element.
+ *
+ * @param input
+ * the real element to notify the eviction policy.
+ * @param triggered
+ * whether a central trigger occurred or not.
+ * @return The number of elements to be deleted from the buffer.
+ */
+ private int centralEviction(IN input, boolean triggered) {
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+ for (EvictionPolicy<IN> evictionPolicy : centralEvictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEviction(input, triggered,
+ deleteOrderForCentralEviction.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+ return currentMaxEviction;
+ }
+
+ /**
+ * This method is used to notify active central eviction policies with a
+ * fake element.
+ *
+ * @param input
+ * the fake element to notify the active central eviction
+ * policies.
+ * @return The number of elements to be deleted from the buffer.
+ */
+ private int centralActiveEviction(Object input) {
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+ for (ActiveEvictionPolicy<IN> evictionPolicy : activeCentralEvictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEvictionWithFakeElement(input,
+ deleteOrderForCentralEviction.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+ return currentMaxEviction;
+ }
+
+ /**
+ * This method is used in central eviction to delete a given number of
+ * elements from the buffer.
+ *
+ * @param numToEvict
+ * number of elements to delete from the virtual central element
+ * buffer.
+ */
+ private void evictElements(int numToEvict) {
+ HashSet<WindowInvokable<IN, OUT>> usedGroups = new HashSet<WindowInvokable<IN, OUT>>();
+ for (; numToEvict > 0; numToEvict--) {
+ WindowInvokable<IN, OUT> currentGroup = deleteOrderForCentralEviction.getFirst();
+ // Do the eviction
+ currentGroup.evictFirst();
+ // Remember groups which possibly have an empty buffer after the
+ // eviction
+ usedGroups.add(currentGroup);
+ try {
+ deleteOrderForCentralEviction.removeFirst();
+ } catch (NoSuchElementException e) {
+ // when buffer is empty, ignore exception and stop deleting
+ break;
+ }
+
+ }
+
+ // Remove groups with empty buffer
+ for (WindowInvokable<IN, OUT> group : usedGroups) {
+ checkForEmptyGroupBuffer(group);
+ }
+ }
+
+ /**
+ * Checks if the element buffer of a given windowing group is empty. If so,
+ * the group will be deleted.
+ *
+ * @param group
+ * The windowing group to be checked and and removed in case its
+ * buffer is empty.
+ */
+ private void checkForEmptyGroupBuffer(WindowInvokable<IN, OUT> group) {
+ if (group.isBufferEmpty()) {
+ windowingGroups.remove(group);
+ }
+ }
+
+ /**
+ * This callback class allows to handle the callbacks done by threads
+ * defined in active trigger policies
+ *
+ * @see ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)
+ */
+ private class WindowingCallback implements ActiveTriggerCallback {
+ private ActiveTriggerPolicy<IN> policy;
+
+ public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
+ this.policy = policy;
+ }
+
+ @Override
+ public void sendFakeElement(Object datapoint) {
+
+ // If central eviction is used, handle it here
+ if (!centralEvictionPolicies.isEmpty()) {
+ evictElements(centralActiveEviction(datapoint));
+ }
+
+ // handle element in groups
+ for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
+ group.processFakeElement(datapoint, policy);
+ checkForEmptyGroupBuffer(group);
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
new file mode 100644
index 0000000..7c8e577
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+
+public class MapInvokable<IN, OUT> extends ChainableInvokable<IN, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ private MapFunction<IN, OUT> mapper;
+
+ public MapInvokable(MapFunction<IN, OUT> mapper) {
+ super(mapper);
+ this.mapper = mapper;
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ while (readNext() != null) {
+ callUserFunctionAndLogException();
+ }
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ collector.collect(mapper.map(nextObject));
+ }
+
+ @Override
+ public void collect(IN record) {
+ nextObject = copy(record);
+ callUserFunctionAndLogException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java
new file mode 100644
index 0000000..c9d9e5a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+
+public class ProjectInvokable<IN, OUT extends Tuple> extends StreamInvokable<IN, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ transient OUT outTuple;
+ TypeSerializer<OUT> outTypeSerializer;
+ int[] fields;
+ int numFields;
+
+ public ProjectInvokable(int[] fields, TypeInformation<OUT> outTypeInformation) {
+ super(null);
+ this.fields = fields;
+ this.numFields = this.fields.length;
+ this.outTypeSerializer = outTypeInformation.createSerializer();
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ while (readNext() != null) {
+ callUserFunctionAndLogException();
+ }
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ for (int i = 0; i < this.numFields; i++) {
+ outTuple.setField(nextRecord.getField(fields[i]), i);
+ }
+ collector.collect(outTuple);
+ }
+
+ @Override
+ public void open(Configuration config) throws Exception {
+ super.open(config);
+ outTuple = outTypeSerializer.createInstance();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
new file mode 100644
index 0000000..e1a56cc
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+
+public class StreamReduceInvokable<IN> extends ChainableInvokable<IN, IN> {
+ private static final long serialVersionUID = 1L;
+
+ protected ReduceFunction<IN> reducer;
+ protected IN currentValue;
+ protected IN nextValue;
+
+ public StreamReduceInvokable(ReduceFunction<IN> reducer) {
+ super(reducer);
+ this.reducer = reducer;
+ currentValue = null;
+ }
+
+ @Override
+ public void invoke() throws Exception {
+ while (readNext() != null) {
+ reduce();
+ }
+ }
+
+ protected void reduce() throws Exception {
+ callUserFunctionAndLogException();
+
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+
+ nextValue = nextObject;
+
+ if (currentValue != null) {
+ currentValue = reducer.reduce(currentValue, nextValue);
+ } else {
+ currentValue = nextValue;
+
+ }
+ collector.collect(currentValue);
+
+ }
+
+ @Override
+ public void collect(IN record) {
+ nextObject = copy(record);
+ callUserFunctionAndLogException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
new file mode 100644
index 0000000..b3fdfe8
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.LinkedList;
+
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+public class WindowGroupReduceInvokable<IN, OUT> extends WindowInvokable<IN, OUT> {
+
+ private static final long serialVersionUID = 1L;
+ GroupReduceFunction<IN, OUT> reducer;
+
+ public WindowGroupReduceInvokable(GroupReduceFunction<IN, OUT> userFunction,
+ LinkedList<TriggerPolicy<IN>> triggerPolicies,
+ LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+ super(userFunction, triggerPolicies, evictionPolicies);
+ this.reducer = userFunction;
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ reducer.reduce(copyBuffer(), collector);
+ }
+
+ public LinkedList<IN> copyBuffer() {
+ LinkedList<IN> copy = new LinkedList<IN>();
+ for (IN element : buffer) {
+ copy.add(copy(element));
+ }
+ return copy;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
new file mode 100644
index 0000000..ea891c9
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
+
+ /**
+ * Auto-generated serial version UID
+ */
+ private static final long serialVersionUID = -8038984294071650730L;
+
+ private LinkedList<TriggerPolicy<IN>> triggerPolicies;
+ private LinkedList<EvictionPolicy<IN>> evictionPolicies;
+ private LinkedList<ActiveTriggerPolicy<IN>> activeTriggerPolicies;
+ private LinkedList<ActiveEvictionPolicy<IN>> activeEvictionPolicies;
+ private LinkedList<Thread> activePolicyTreads;
+ protected LinkedList<IN> buffer;
+ private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
+
+ /**
+ * This constructor created a windowing invokable using trigger and eviction
+ * policies.
+ *
+ * @param userFunction
+ * The user defined {@link ReduceFunction}
+ * @param triggerPolicies
+ * A list of {@link TriggerPolicy}s and/or
+ * {@link ActiveTriggerPolicy}s
+ * @param evictionPolicies
+ * A list of {@link EvictionPolicy}s and/or
+ * {@link ActiveEvictionPolicy}s
+ */
+ public WindowInvokable(Function userFunction, LinkedList<TriggerPolicy<IN>> triggerPolicies,
+ LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+ super(userFunction);
+
+ this.triggerPolicies = triggerPolicies;
+ this.evictionPolicies = evictionPolicies;
+
+ activeTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
+ for (TriggerPolicy<IN> tp : triggerPolicies) {
+ if (tp instanceof ActiveTriggerPolicy) {
+ activeTriggerPolicies.add((ActiveTriggerPolicy<IN>) tp);
+ }
+ }
+
+ activeEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
+ for (EvictionPolicy<IN> ep : evictionPolicies) {
+ if (ep instanceof ActiveEvictionPolicy) {
+ activeEvictionPolicies.add((ActiveEvictionPolicy<IN>) ep);
+ }
+ }
+
+ this.activePolicyTreads = new LinkedList<Thread>();
+ this.buffer = new LinkedList<IN>();
+ this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ }
+
+ @Override
+ public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
+ super.open(parameters);
+ for (ActiveTriggerPolicy<IN> tp : activeTriggerPolicies) {
+ Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
+ if (target != null) {
+ Thread thread = new Thread(target);
+ activePolicyTreads.add(thread);
+ thread.start();
+ }
+ }
+ };
+
+ /**
+ * This class allows the active trigger threads to call back and push fake
+ * elements at any time.
+ */
+ private class WindowingCallback implements ActiveTriggerCallback {
+ private ActiveTriggerPolicy<IN> policy;
+
+ public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
+ this.policy = policy;
+ }
+
+ @Override
+ public void sendFakeElement(Object datapoint) {
+ processFakeElement(datapoint, this.policy);
+ }
+
+ }
+
+ @Override
+ public void invoke() throws Exception {
+
+ // Prevent empty data streams
+ if (readNext() == null) {
+ throw new RuntimeException("DataStream must not be empty");
+ }
+
+ // Continuously run
+ while (nextRecord != null) {
+ processRealElement(nextRecord.getObject());
+
+ // Load next StreamRecord
+ readNext();
+ }
+
+ // Stop all remaining threads from policies
+ for (Thread t : activePolicyTreads) {
+ t.interrupt();
+ }
+
+ // finally trigger the buffer.
+ emitFinalWindow(null);
+
+ }
+
+ /**
+ * This method gets called in case of an grouped windowing in case central
+ * trigger occurred and the arriving element causing the trigger is not part
+ * of this group.
+ *
+ * Remark: This is NOT the same as
+ * {@link WindowInvokable#processFakeElement(Object, TriggerPolicy)}! Here
+ * the eviction using active policies takes place after the call to the UDF.
+ * Usually it is done before when fake elements get submitted. This special
+ * behaviour is needed to allow the {@link GroupedWindowInvokable} to send
+ * central triggers to all groups, even if the current element does not
+ * belong to the group.
+ *
+ * @param input
+ * a fake input element
+ * @param policies
+ * the list of policies which caused the call with this fake
+ * element
+ */
+ protected synchronized void externalTriggerFakeElement(IN input,
+ List<TriggerPolicy<IN>> policies) {
+
+ // Set the current triggers
+ currentTriggerPolicies.addAll(policies);
+
+ // emit
+ callUserFunctionAndLogException();
+
+ // clear the flag collection
+ currentTriggerPolicies.clear();
+
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+ for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+
+ for (int i = 0; i < currentMaxEviction; i++) {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // In case no more elements are in the buffer:
+ // Prevent failure and stop deleting.
+ break;
+ }
+ }
+ }
+
+ /**
+ * This method processed an arrived fake element The method is synchronized
+ * to ensure that it cannot interleave with
+ * {@link WindowInvokable#processRealElement(Object)}
+ *
+ * @param input
+ * a fake input element
+ * @param currentPolicy
+ * the policy which produced this fake element
+ */
+ protected synchronized void processFakeElement(Object input, TriggerPolicy<IN> currentPolicy) {
+
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+ for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+
+ for (int i = 0; i < currentMaxEviction; i++) {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // In case no more elements are in the buffer:
+ // Prevent failure and stop deleting.
+ break;
+ }
+ }
+
+ // Set the current trigger
+ currentTriggerPolicies.add(currentPolicy);
+
+ // emit
+ callUserFunctionAndLogException();
+
+ // clear the flag collection
+ currentTriggerPolicies.clear();
+ }
+
+ /**
+ * This method processed an arrived real element The method is synchronized
+ * to ensure that it cannot interleave with
+ * {@link WindowInvokable#processFakeElement(Object)}.
+ *
+ * @param input
+ * a real input element
+ * @param triggerPolicies
+ * Allows to set trigger policies which are maintained
+ * externally. This is the case for central policies in
+ * {@link GroupedWindowInvokable}.
+ */
+ protected synchronized void processRealElement(IN input, List<TriggerPolicy<IN>> triggerPolicies) {
+ this.currentTriggerPolicies.addAll(triggerPolicies);
+ processRealElement(input);
+ }
+
+ /**
+ * This method processed an arrived real element The method is synchronized
+ * to ensure that it cannot interleave with
+ * {@link WindowInvokable#processFakeElement(Object)}
+ *
+ * @param input
+ * a real input element
+ */
+ protected synchronized void processRealElement(IN input) {
+
+ // Run the precalls to detect missed windows
+ for (ActiveTriggerPolicy<IN> trigger : activeTriggerPolicies) {
+ // Remark: In case multiple active triggers are present the ordering
+ // of the different fake elements returned by this triggers becomes
+ // a problem. This might lead to unexpected results...
+ // Should we limit the number of active triggers to 0 or 1?
+ Object[] result = trigger.preNotifyTrigger(input);
+ for (Object in : result) {
+ processFakeElement(in, trigger);
+ }
+ }
+
+ // Remember if a trigger occurred
+ boolean isTriggered = false;
+
+ // Process the triggers
+ for (TriggerPolicy<IN> triggerPolicy : triggerPolicies) {
+ if (triggerPolicy.notifyTrigger(input)) {
+ currentTriggerPolicies.add(triggerPolicy);
+ }
+ }
+
+ // call user function
+ if (!currentTriggerPolicies.isEmpty()) {
+ // emit
+ callUserFunctionAndLogException();
+
+ // clear the flag collection
+ currentTriggerPolicies.clear();
+
+ // remember trigger
+ isTriggered = true;
+ }
+
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+
+ for (EvictionPolicy<IN> evictionPolicy : evictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEviction(input, isTriggered, buffer.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+
+ for (int i = 0; i < currentMaxEviction; i++) {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // In case no more elements are in the buffer:
+ // Prevent failure and stop deleting.
+ break;
+ }
+ }
+
+ // Add the current element to the buffer
+ buffer.add(input);
+
+ }
+
+ /**
+ * This method removes the first element from the element buffer. It is used
+ * to provide central evictions in {@link GroupedWindowInvokable}
+ */
+ protected synchronized void evictFirst() {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // ignore exception
+ }
+ }
+
+ /**
+ * This method returns whether the element buffer is empty or not. It is
+ * used to figure out if a group can be deleted or not when
+ * {@link GroupedWindowInvokable} is used.
+ *
+ * @return true in case the buffer is empty otherwise false.
+ */
+ protected boolean isBufferEmpty() {
+ return buffer.isEmpty();
+ }
+
+ /**
+ * This method does the final reduce at the end of the stream and emits the
+ * result.
+ *
+ * @param centralTriggerPolicies
+ * Allows to set trigger policies which are maintained
+ * externally. This is the case for central policies in
+ * {@link GroupedWindowInvokable}.
+ */
+ protected void emitFinalWindow(List<TriggerPolicy<IN>> centralTriggerPolicies) {
+ if (!buffer.isEmpty()) {
+ currentTriggerPolicies.clear();
+
+ if (centralTriggerPolicies != null) {
+ currentTriggerPolicies.addAll(centralTriggerPolicies);
+ }
+
+ for (TriggerPolicy<IN> policy : triggerPolicies) {
+ currentTriggerPolicies.add(policy);
+ }
+
+ callUserFunctionAndLogException();
+ }
+ }
+
+}
[32/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FilterTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FilterTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FilterTest.java
deleted file mode 100644
index 403dd17..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FilterTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class FilterTest implements Serializable {
- private static final long serialVersionUID = 1L;
-
- static class MyFilter implements FilterFunction<Integer> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public boolean filter(Integer value) throws Exception {
- return value % 2 == 0;
- }
- }
-
- @Test
- public void test() {
- FilterInvokable<Integer> invokable = new FilterInvokable<Integer>(new MyFilter());
-
- List<Integer> expected = Arrays.asList(2, 4, 6);
- List<Integer> actual = MockContext.createAndExecute(invokable, Arrays.asList(1, 2, 3, 4, 5, 6, 7));
-
- assertEquals(expected, actual);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FlatMapTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FlatMapTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FlatMapTest.java
deleted file mode 100644
index 7424e21..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FlatMapTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.streaming.util.MockContext;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class FlatMapTest {
-
- public static final class MyFlatMap implements FlatMapFunction<Integer, Integer> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(Integer value, Collector<Integer> out) throws Exception {
- if (value % 2 == 0) {
- out.collect(value);
- out.collect(value * value);
- }
- }
- }
-
- @Test
- public void flatMapTest() {
- FlatMapInvokable<Integer, Integer> invokable = new FlatMapInvokable<Integer, Integer>(new MyFlatMap());
-
- List<Integer> expected = Arrays.asList(2, 4, 4, 16, 6, 36, 8, 64);
- List<Integer> actual = MockContext.createAndExecute(invokable, Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8));
-
- assertEquals(expected, actual);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java
deleted file mode 100755
index ce47c67..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class GroupedReduceInvokableTest {
-
- private static class MyReducer implements ReduceFunction<Integer> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
-
- }
-
- @Test
- public void test() {
- GroupedReduceInvokable<Integer> invokable1 = new GroupedReduceInvokable<Integer>(
- new MyReducer(), new KeySelector<Integer, Integer>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(Integer value) throws Exception {
- return value;
- }
- });
-
- List<Integer> expected = Arrays.asList(1, 2, 2, 4, 3);
- List<Integer> actual = MockContext.createAndExecute(invokable1,
- Arrays.asList(1, 1, 2, 2, 3));
-
- assertEquals(expected, actual);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
deleted file mode 100644
index f38d5c1..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
+++ /dev/null
@@ -1,574 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
-import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class GroupedWindowInvokableTest {
-
- KeySelector<Tuple2<Integer, String>, ?> keySelector = new KeySelector<Tuple2<Integer, String>, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String getKey(Tuple2<Integer, String> value) throws Exception {
- return value.f1;
- }
- };
-
- /**
- * Tests that illegal arguments result in failure. The following cases are
- * tested: 1) having no trigger 2) having no eviction 3) having neither
- * eviction nor trigger 4) having both, central and distributed eviction.
- */
- @Test
- public void testGroupedWindowInvokableFailTest() {
-
- // create dummy reduce function
- ReduceFunction<Object> userFunction = new ReduceFunction<Object>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Object reduce(Object value1, Object value2) throws Exception {
- return null;
- }
- };
-
- // create dummy keySelector
- KeySelector<Object, Object> keySelector = new KeySelector<Object, Object>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Object getKey(Object value) throws Exception {
- return null;
- }
- };
-
- // create policy lists
- LinkedList<CloneableEvictionPolicy<Object>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<Object>>();
- LinkedList<CloneableTriggerPolicy<Object>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<Object>>();
- LinkedList<EvictionPolicy<Object>> centralEvictionPolicies = new LinkedList<EvictionPolicy<Object>>();
- LinkedList<TriggerPolicy<Object>> centralTriggerPolicies = new LinkedList<TriggerPolicy<Object>>();
-
- // empty trigger and policy lists should fail
- try {
- new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
- distributedTriggerPolicies, distributedEvictionPolicies,
- centralTriggerPolicies, centralEvictionPolicies);
- fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (1)");
- } catch (UnsupportedOperationException e) {
- // that's the expected case
- }
-
- // null for trigger and policy lists should fail
- try {
- new GroupedWindowInvokable<Object, Object>(userFunction, keySelector, null, null, null,
- null);
- fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (2)");
- } catch (UnsupportedOperationException e) {
- // that's the expected case
- }
-
- // empty eviction should still fail
- centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
- distributedTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
- try {
- new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
- distributedTriggerPolicies, distributedEvictionPolicies,
- centralTriggerPolicies, centralEvictionPolicies);
- fail("Creating instance without any eviction policy should cause an UnsupportedOperationException but didn't. (3)");
- } catch (UnsupportedOperationException e) {
- // that's the expected case
- }
-
- // empty trigger should still fail
- centralTriggerPolicies.clear();
- distributedTriggerPolicies.clear();
- centralEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
- try {
- new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
- distributedTriggerPolicies, distributedEvictionPolicies,
- centralTriggerPolicies, centralEvictionPolicies);
- fail("Creating instance without any trigger policy should cause an UnsupportedOperationException but didn't. (4)");
- } catch (UnsupportedOperationException e) {
- // that's the expected case
- }
-
- // having both, central and distributed eviction, at the same time
- // should fail
- centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
- distributedEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
- try {
- new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
- distributedTriggerPolicies, distributedEvictionPolicies,
- centralTriggerPolicies, centralEvictionPolicies);
- fail("Creating instance with central and distributed eviction should cause an UnsupportedOperationException but didn't. (4)");
- } catch (UnsupportedOperationException e) {
- // that's the expected case
- }
-
- }
-
- /**
- * Test for not active distributed triggers with single field
- */
- @Test
- public void testGroupedWindowInvokableDistributedTriggerSimple() {
- List<Integer> inputs = new ArrayList<Integer>();
- inputs.add(1);
- inputs.add(1);
- inputs.add(5);
- inputs.add(5);
- inputs.add(5);
- inputs.add(1);
- inputs.add(1);
- inputs.add(5);
- inputs.add(1);
- inputs.add(5);
-
- List<Integer> expectedDistributedEviction = new ArrayList<Integer>();
- expectedDistributedEviction.add(15);
- expectedDistributedEviction.add(3);
- expectedDistributedEviction.add(3);
- expectedDistributedEviction.add(15);
-
- List<Integer> expectedCentralEviction = new ArrayList<Integer>();
- expectedCentralEviction.add(2);
- expectedCentralEviction.add(5);
- expectedCentralEviction.add(15);
- expectedCentralEviction.add(2);
- expectedCentralEviction.add(5);
- expectedCentralEviction.add(2);
- expectedCentralEviction.add(5);
- expectedCentralEviction.add(1);
- expectedCentralEviction.add(5);
-
- LinkedList<CloneableTriggerPolicy<Integer>> triggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
- // Trigger on every 2nd element, but the first time after the 3rd
- triggers.add(new CountTriggerPolicy<Integer>(2, -1));
-
- LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
- // On every 2nd element, remove the oldest 2 elements, but the first
- // time after the 3rd element
- evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
- LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
-
- ReduceFunction<Integer> reduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(Integer value) {
- return value;
- }
- };
-
- GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
- reduceFunction, keySelector, triggers, evictions, centralTriggers, null);
-
- List<Integer> result = MockContext.createAndExecute(invokable, inputs);
-
- List<Integer> actual = new LinkedList<Integer>();
- for (Integer current : result) {
- actual.add(current);
- }
-
- assertEquals(new HashSet<Integer>(expectedDistributedEviction),
- new HashSet<Integer>(actual));
- assertEquals(expectedDistributedEviction.size(), actual.size());
-
- // Run test with central eviction
- triggers.clear();
- centralTriggers.add(new CountTriggerPolicy<Integer>(2, -1));
- LinkedList<EvictionPolicy<Integer>> centralEvictions = new LinkedList<EvictionPolicy<Integer>>();
- centralEvictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
- invokable = new GroupedWindowInvokable<Integer, Integer>(reduceFunction, keySelector,
- triggers, null, centralTriggers, centralEvictions);
-
- result = MockContext.createAndExecute(invokable, inputs);
- actual = new LinkedList<Integer>();
- for (Integer current : result) {
- actual.add(current);
- }
-
- assertEquals(new HashSet<Integer>(expectedCentralEviction), new HashSet<Integer>(actual));
- assertEquals(expectedCentralEviction.size(), actual.size());
- }
-
- /**
- * Test for non active distributed triggers with separated key field
- */
- @Test
- public void testGroupedWindowInvokableDistributedTriggerComplex() {
- List<Tuple2<Integer, String>> inputs2 = new ArrayList<Tuple2<Integer, String>>();
- inputs2.add(new Tuple2<Integer, String>(1, "a"));
- inputs2.add(new Tuple2<Integer, String>(0, "b"));
- inputs2.add(new Tuple2<Integer, String>(2, "a"));
- inputs2.add(new Tuple2<Integer, String>(-1, "a"));
- inputs2.add(new Tuple2<Integer, String>(-2, "a"));
- inputs2.add(new Tuple2<Integer, String>(10, "a"));
- inputs2.add(new Tuple2<Integer, String>(2, "b"));
- inputs2.add(new Tuple2<Integer, String>(1, "a"));
-
- List<Tuple2<Integer, String>> expected2 = new ArrayList<Tuple2<Integer, String>>();
- expected2.add(new Tuple2<Integer, String>(-1, "a"));
- expected2.add(new Tuple2<Integer, String>(-2, "a"));
- expected2.add(new Tuple2<Integer, String>(0, "b"));
-
- LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
- // Trigger on every 2nd element, but the first time after the 3rd
- triggers.add(new CountTriggerPolicy<Tuple2<Integer, String>>(3));
-
- LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
- // On every 2nd element, remove the oldest 2 elements, but the first
- // time after the 3rd element
- evictions.add(new TumblingEvictionPolicy<Tuple2<Integer, String>>());
-
- LinkedList<TriggerPolicy<Tuple2<Integer, String>>> centralTriggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
-
- GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable2 = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
- new ReduceFunction<Tuple2<Integer, String>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
- Tuple2<Integer, String> value2) throws Exception {
- if (value1.f0 <= value2.f0) {
- return value1;
- } else {
- return value2;
- }
- }
- }, keySelector, triggers, evictions, centralTriggers, null);
-
- List<Tuple2<Integer, String>> result = MockContext.createAndExecute(invokable2, inputs2);
-
- List<Tuple2<Integer, String>> actual2 = new LinkedList<Tuple2<Integer, String>>();
- for (Tuple2<Integer, String> current : result) {
- actual2.add(current);
- }
-
- assertEquals(new HashSet<Tuple2<Integer, String>>(expected2),
- new HashSet<Tuple2<Integer, String>>(actual2));
- assertEquals(expected2.size(), actual2.size());
- }
-
- /**
- * Test for active centralized trigger
- */
- @Test
- public void testGroupedWindowInvokableCentralActiveTrigger() {
-
- List<Tuple2<Integer, String>> inputs = new ArrayList<Tuple2<Integer, String>>();
- inputs.add(new Tuple2<Integer, String>(1, "a"));
- inputs.add(new Tuple2<Integer, String>(1, "b"));
- inputs.add(new Tuple2<Integer, String>(1, "c"));
- inputs.add(new Tuple2<Integer, String>(2, "a"));
- inputs.add(new Tuple2<Integer, String>(2, "b"));
- inputs.add(new Tuple2<Integer, String>(2, "c"));
- inputs.add(new Tuple2<Integer, String>(2, "b"));
- inputs.add(new Tuple2<Integer, String>(2, "a"));
- inputs.add(new Tuple2<Integer, String>(2, "c"));
- inputs.add(new Tuple2<Integer, String>(3, "c"));
- inputs.add(new Tuple2<Integer, String>(3, "a"));
- inputs.add(new Tuple2<Integer, String>(3, "b"));
- inputs.add(new Tuple2<Integer, String>(4, "a"));
- inputs.add(new Tuple2<Integer, String>(4, "b"));
- inputs.add(new Tuple2<Integer, String>(4, "c"));
- inputs.add(new Tuple2<Integer, String>(5, "c"));
- inputs.add(new Tuple2<Integer, String>(5, "a"));
- inputs.add(new Tuple2<Integer, String>(5, "b"));
- inputs.add(new Tuple2<Integer, String>(10, "b"));
- inputs.add(new Tuple2<Integer, String>(10, "a"));
- inputs.add(new Tuple2<Integer, String>(10, "c"));
- inputs.add(new Tuple2<Integer, String>(11, "a"));
- inputs.add(new Tuple2<Integer, String>(11, "a"));
- inputs.add(new Tuple2<Integer, String>(11, "c"));
- inputs.add(new Tuple2<Integer, String>(11, "c"));
- inputs.add(new Tuple2<Integer, String>(11, "b"));
- inputs.add(new Tuple2<Integer, String>(11, "b"));
-
- // Expected result:
- // For each group (a,b and c):
- // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
- // 12-12-5-10-32
-
- List<Tuple2<Integer, String>> expected = new ArrayList<Tuple2<Integer, String>>();
- expected.add(new Tuple2<Integer, String>(12, "a"));
- expected.add(new Tuple2<Integer, String>(12, "b"));
- expected.add(new Tuple2<Integer, String>(12, "c"));
- expected.add(new Tuple2<Integer, String>(12, "a"));
- expected.add(new Tuple2<Integer, String>(12, "b"));
- expected.add(new Tuple2<Integer, String>(12, "c"));
- expected.add(new Tuple2<Integer, String>(5, "a"));
- expected.add(new Tuple2<Integer, String>(5, "b"));
- expected.add(new Tuple2<Integer, String>(5, "c"));
- expected.add(new Tuple2<Integer, String>(10, "a"));
- expected.add(new Tuple2<Integer, String>(10, "b"));
- expected.add(new Tuple2<Integer, String>(10, "c"));
- expected.add(new Tuple2<Integer, String>(32, "a"));
- expected.add(new Tuple2<Integer, String>(32, "b"));
- expected.add(new Tuple2<Integer, String>(32, "c"));
-
- Timestamp<Tuple2<Integer, String>> myTimeStamp = new Timestamp<Tuple2<Integer, String>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Tuple2<Integer, String> value) {
- return value.f0;
- }
- };
-
- TimestampWrapper<Tuple2<Integer, String>> myTimeStampWrapper = new TimestampWrapper<Tuple2<Integer, String>>(
- myTimeStamp, 1);
-
- ReduceFunction<Tuple2<Integer, String>> myReduceFunction = new ReduceFunction<Tuple2<Integer, String>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
- Tuple2<Integer, String> value2) throws Exception {
- return new Tuple2<Integer, String>(value1.f0 + value2.f0, value1.f1);
- }
- };
-
- LinkedList<TriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
- // Trigger every 2 time units but delay the first trigger by 2 (First
- // trigger after 4, then every 2)
- triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
-
- LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
- // Always delete all elements older then 4
- evictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
-
- LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
-
- GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
- myReduceFunction, keySelector, distributedTriggers, evictions, triggers, null);
-
- ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
- for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
- new HashSet<Tuple2<Integer, String>>(result));
- assertEquals(expected.size(), result.size());
-
- // repeat the test with central eviction. The result should be the same.
- triggers.clear();
- triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
- evictions.clear();
- LinkedList<EvictionPolicy<Tuple2<Integer, String>>> centralEvictions = new LinkedList<EvictionPolicy<Tuple2<Integer, String>>>();
- centralEvictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
-
- invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
- myReduceFunction, keySelector, distributedTriggers, evictions, triggers,
- centralEvictions);
-
- result = new ArrayList<Tuple2<Integer, String>>();
- for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
- new HashSet<Tuple2<Integer, String>>(result));
- assertEquals(expected.size(), result.size());
- }
-
- /**
- * Test for multiple centralized trigger
- */
- @Test
- public void testGroupedWindowInvokableMultipleCentralTrigger() {
- LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
- triggers.add(new CountTriggerPolicy<Integer>(8));
- triggers.add(new CountTriggerPolicy<Integer>(5));
-
- LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
- // The active wrapper causes eviction even on (fake) elements which
- // triggered, but does not belong to the group.
- evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
- new TumblingEvictionPolicy<Integer>()));
-
- LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
-
- List<Integer> inputs = new ArrayList<Integer>();
- inputs.add(1);
- inputs.add(2);
- inputs.add(2);
- inputs.add(2);
- inputs.add(1);
- // 1st Trigger: 2;6
- inputs.add(2);
- inputs.add(1);
- inputs.add(2);
- // 2nd Trigger: 1;4
- inputs.add(2);
- inputs.add(1);
- // Final: 1,2
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(2);
- expected.add(6);
- expected.add(4);
- expected.add(1);
- expected.add(2);
- expected.add(1);
-
- ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
- myReduceFunction, new KeySelector<Integer, Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(Integer value) {
- return value;
- }
- }, distributedTriggers, evictions, triggers, null);
-
- ArrayList<Integer> result = new ArrayList<Integer>();
- for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
- assertEquals(expected.size(), result.size());
- }
-
- /**
- * Test for combination of centralized trigger and distributed trigger at
- * the same time
- */
- @Test
- public void testGroupedWindowInvokableCentralAndDistrTrigger() {
- LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
- triggers.add(new CountTriggerPolicy<Integer>(8));
- triggers.add(new CountTriggerPolicy<Integer>(5));
-
- LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
- // The active wrapper causes eviction even on (fake) elements which
- // triggered, but does not belong to the group.
- evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
- new TumblingEvictionPolicy<Integer>()));
-
- LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
- distributedTriggers.add(new CountTriggerPolicy<Integer>(2));
-
- List<Integer> inputs = new ArrayList<Integer>();
- inputs.add(1);
- inputs.add(2);
- inputs.add(2);
- // local on 2 => 4
- inputs.add(2);
- inputs.add(1);
- // local on 1 => 2
- // and 1st Central: 2;2
- // SUMS up to 2;2
- inputs.add(2);
- // local on 2 => 2
- inputs.add(1);
- inputs.add(2);
- // 2nd Central: 1;2
- inputs.add(2);
- inputs.add(1);
- // Final: 1,2
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(4);
- expected.add(2);
- expected.add(2);
- expected.add(2);
- expected.add(1);
- expected.add(2);
- expected.add(1);
- expected.add(2);
-
- ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
- myReduceFunction, new KeySelector<Integer, Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(Integer value) {
- return value;
- }
- }, distributedTriggers, evictions, triggers, null);
-
- ArrayList<Integer> result = new ArrayList<Integer>();
- for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
- assertEquals(expected.size(), result.size());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/MapTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/MapTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/MapTest.java
deleted file mode 100644
index 5390ec9..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/MapTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class MapTest {
-
- private static class Map implements MapFunction<Integer, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public String map(Integer value) throws Exception {
- return "+" + (value + 1);
- }
- }
-
- @Test
- public void mapInvokableTest() {
- MapInvokable<Integer, String> invokable = new MapInvokable<Integer, String>(new Map());
-
- List<String> expectedList = Arrays.asList("+2", "+3", "+4");
- List<String> actualList = MockContext.createAndExecute(invokable, Arrays.asList(1, 2, 3));
-
- assertEquals(expectedList, actualList);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/ProjectTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/ProjectTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/ProjectTest.java
deleted file mode 100644
index 11c44cd..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/ProjectTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.datastream.StreamProjection;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class ProjectTest implements Serializable {
- private static final long serialVersionUID = 1L;
-
- @Test
- public void test() {
-
- TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
- .getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b",
- 4));
-
- int[] fields = new int[] { 4, 4, 3 };
- Class<?>[] classes = new Class<?>[] { Integer.class, Integer.class, String.class };
-
- @SuppressWarnings("unchecked")
- ProjectInvokable<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> invokable = new ProjectInvokable<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
- fields,
- (TypeInformation<Tuple3<Integer, Integer, String>>) StreamProjection
- .extractFieldTypes(fields, classes, inType));
-
- List<Tuple5<Integer, String, Integer, String, Integer>> input = new ArrayList<Tuple5<Integer, String, Integer, String, Integer>>();
- input.add(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));
- input.add(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2));
- input.add(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2));
- input.add(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7));
-
- List<Tuple3<Integer, Integer, String>> expected = new ArrayList<Tuple3<Integer, Integer, String>>();
- expected.add(new Tuple3<Integer, Integer, String>(4, 4, "b"));
- expected.add(new Tuple3<Integer, Integer, String>(2, 2, "c"));
- expected.add(new Tuple3<Integer, Integer, String>(2, 2, "c"));
- expected.add(new Tuple3<Integer, Integer, String>(7, 7, "a"));
-
- assertEquals(expected, MockContext.createAndExecute(invokable, input));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceTest.java
deleted file mode 100755
index ae866e6..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class StreamReduceTest {
-
- private static class MyReducer implements ReduceFunction<Integer>{
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1+value2;
- }
-
- }
-
- @Test
- public void test() {
- StreamReduceInvokable<Integer> invokable1 = new StreamReduceInvokable<Integer>(
- new MyReducer());
-
- List<Integer> expected = Arrays.asList(1,2,4,7,10);
- List<Integer> actual = MockContext.createAndExecute(invokable1,
- Arrays.asList(1, 1, 2, 3, 3));
-
- assertEquals(expected, actual);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
deleted file mode 100644
index 83b4596..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class WindowInvokableTest {
-
- /**
- * Test case equal to {@link WindowReduceInvokableTest}
- */
- @Test
- public void testWindowInvokableWithTimePolicy() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- inputs.add(1);
- inputs.add(2);
- inputs.add(2);
- inputs.add(3);
- inputs.add(4);
- inputs.add(5);
- inputs.add(10);
- inputs.add(11);
- inputs.add(11);
- // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
- // 12-12-5-10-32
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(12);
- expected.add(12);
- expected.add(5);
- expected.add(10);
- expected.add(32);
-
- Timestamp<Integer> myTimeStamp = new Timestamp<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Integer value) {
- return value;
- }
- };
-
- ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
- // Trigger every 2 time units but delay the first trigger by 2 (First
- // trigger after 4, then every 2)
- triggers.add(new TimeTriggerPolicy<Integer>(2L, new TimestampWrapper<Integer>(myTimeStamp,
- 1), 2L));
-
- LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
- // Always delete all elements older then 4
- evictions.add(new TimeEvictionPolicy<Integer>(4L, new TimestampWrapper<Integer>(
- myTimeStamp, 1)));
-
- WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
- myReduceFunction, triggers, evictions);
-
- ArrayList<Integer> result = new ArrayList<Integer>();
- for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(expected, result);
- }
-
- /**
- * Test case equal to {@link BatchReduceTest}
- */
- @Test
- public void testWindowInvokableWithCountPolicy() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- for (Integer i = 1; i <= 10; i++) {
- inputs.add(i);
- }
-
- ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- /*
- * The following setup reassembles the batch size 3 and the slide size 2
- * of the BatchReduceInvokable.
- */
- LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
- // Trigger on every 2nd element, but the first time after the 3rd
- triggers.add(new CountTriggerPolicy<Integer>(2, -1));
-
- LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
- // On every 2nd element, remove the oldest 2 elements, but the first
- // time after the 3rd element
- evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
- WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
- myReduceFunction, triggers, evictions);
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(6);
- expected.add(12);
- expected.add(18);
- expected.add(24);
- expected.add(19);
- List<Integer> result = new ArrayList<Integer>();
- for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
- assertEquals(expected, result);
-
- /*
- * Begin test part 2
- */
-
- List<Integer> inputs2 = new ArrayList<Integer>();
- inputs2.add(1);
- inputs2.add(2);
- inputs2.add(-5); // changed this value to make sure it is excluded from
- // the result
- inputs2.add(-3);
- inputs2.add(-4);
-
- myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- if (value1 <= value2) {
- return value1;
- } else {
- return value2;
- }
- };
- };
-
- /*
- * The following setup reassembles the batch size 2 and the slide size 3
- * of the BatchReduceInvokable.
- */
- triggers = new LinkedList<TriggerPolicy<Integer>>();
- // Trigger after every 3rd element, but the first time after the 2nd
- triggers.add(new CountTriggerPolicy<Integer>(3, 1));
-
- evictions = new LinkedList<EvictionPolicy<Integer>>();
- // On every 3rd element, remove the oldest 3 elements, but the first
- // time after on the 5th element
- evictions.add(new CountEvictionPolicy<Integer>(3, 3, -1));
-
- WindowInvokable<Integer, Integer> invokable2 = new WindowReduceInvokable<Integer>(
- myReduceFunction, triggers, evictions);
-
- List<Integer> expected2 = new ArrayList<Integer>();
- expected2.add(1);
- expected2.add(-4);
-
- result = new ArrayList<Integer>();
- for (Integer t : MockContext.createAndExecute(invokable2, inputs2)) {
- result.add(t);
- }
-
- assertEquals(expected2, result);
-
- }
-
- @Test
- public void testWindowInvokableWithMultiplePolicies() {
- LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
- triggers.add(new CountTriggerPolicy<Integer>(2));
- triggers.add(new CountTriggerPolicy<Integer>(3));
-
- LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
- evictions.add(new CountEvictionPolicy<Integer>(2, 2));
- evictions.add(new CountEvictionPolicy<Integer>(3, 3));
-
- List<Integer> inputs = new ArrayList<Integer>();
- for (Integer i = 1; i <= 10; i++) {
- inputs.add(i);
- }
- /**
- * <code>
- * VAL: 1,2,3,4,5,6,7,8,9,10
- * TR1: | | | | |
- * TR2: | | |
- * EV1: 2 2 2 2 2
- * EV2: 3 3 3
- * </code>
- */
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(3);
- expected.add(3);
- expected.add(4);
- expected.add(11);
- expected.add(15);
- expected.add(9);
- expected.add(10);
-
- ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
- myReduceFunction, triggers, evictions);
-
- ArrayList<Integer> result = new ArrayList<Integer>();
- for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(expected, result);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamrecord/UIDTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamrecord/UIDTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamrecord/UIDTest.java
deleted file mode 100644
index b044d88..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamrecord/UIDTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamrecord;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.flink.runtime.util.DataInputDeserializer;
-import org.apache.flink.runtime.util.DataOutputSerializer;
-import org.junit.Test;
-
-public class UIDTest {
-
- //TODO fix with matching DataOutputStream and DataOutputView
- @Test
- public void test() throws IOException {
- DataOutputSerializer out = new DataOutputSerializer(64);
-
- UID id = new UID(3);
- id.write(out);
-
- ByteBuffer buff = out.wrapAsByteBuffer();
-
-
- DataInputDeserializer in = new DataInputDeserializer(buff);
-
- UID id2 = new UID();
- id2.read(in);
-
- assertEquals(id.getChannelId(), id2.getChannelId());
- assertArrayEquals(id.getGeneratedId(), id2.getGeneratedId());
- assertArrayEquals(id.getId(), id2.getId());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/MockRecordWriter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/MockRecordWriter.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/MockRecordWriter.java
deleted file mode 100755
index a40048e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/MockRecordWriter.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-import java.util.ArrayList;
-
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.runtime.operators.DataSourceTask;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
-
-public class MockRecordWriter extends RecordWriter<SerializationDelegate<StreamRecord<Tuple1<Integer>>>> {
-
- public ArrayList<Integer> emittedRecords;
-
- public MockRecordWriter(DataSourceTask<?> inputBase, Class<StreamRecord<Tuple1<Integer>>> outputClass) {
- super(inputBase.getEnvironment().getWriter(0));
- }
-
- public boolean initList() {
- emittedRecords = new ArrayList<Integer>();
- return true;
- }
-
- @Override
- public void emit(SerializationDelegate<StreamRecord<Tuple1<Integer>>> record) {
- emittedRecords.add(record.getInstance().getObject().f0);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/StreamVertexTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/StreamVertexTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/StreamVertexTest.java
deleted file mode 100644
index b103d84..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/StreamVertexTest.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.co.CoMapFunction;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.util.TestStreamEnvironment;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class StreamVertexTest {
-
- private static Map<Integer, Integer> data = new HashMap<Integer, Integer>();
-
- public static class MySource implements SourceFunction<Tuple1<Integer>> {
- private static final long serialVersionUID = 1L;
-
- private Tuple1<Integer> tuple = new Tuple1<Integer>(0);
-
- @Override
- public void invoke(Collector<Tuple1<Integer>> collector) throws Exception {
- for (int i = 0; i < 10; i++) {
- tuple.f0 = i;
- collector.collect(tuple);
- }
- }
- }
-
- public static class MyTask extends RichMapFunction<Tuple1<Integer>, Tuple2<Integer, Integer>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<Integer, Integer> map(Tuple1<Integer> value) throws Exception {
- Integer i = value.f0;
- return new Tuple2<Integer, Integer>(i, i + 1);
- }
- }
-
- public static class MySink implements SinkFunction<Tuple2<Integer, Integer>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(Tuple2<Integer, Integer> tuple) {
- Integer k = tuple.getField(0);
- Integer v = tuple.getField(1);
- data.put(k, v);
- }
- }
-
- @SuppressWarnings("unused")
- private static final int PARALLELISM = 1;
- private static final int SOURCE_PARALELISM = 1;
- private static final long MEMORYSIZE = 32;
-
- @Test
- public void wrongJobGraph() {
- LocalStreamEnvironment env = StreamExecutionEnvironment
- .createLocalEnvironment(SOURCE_PARALELISM);
-
-
- try {
- env.fromCollection(null);
- fail();
- } catch (NullPointerException e) {
- }
-
- try {
- env.fromElements();
- fail();
- } catch (IllegalArgumentException e) {
- }
-
- try {
- env.generateSequence(-10, -30);
- fail();
- } catch (IllegalArgumentException e) {
- }
-
- try {
- env.setBufferTimeout(-10);
- fail();
- } catch (IllegalArgumentException e) {
- }
-
- try {
- env.generateSequence(1, 10).project(2);
- fail();
- } catch (RuntimeException e) {
- }
- }
-
- private static class CoMap implements CoMapFunction<String, Long, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public String map1(String value) {
- return value;
- }
-
- @Override
- public String map2(Long value) {
- return value.toString();
- }
- }
-
- static HashSet<String> resultSet;
-
- private static class SetSink implements SinkFunction<String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(String value) {
- resultSet.add(value);
- }
- }
-
- @Test
- public void coTest() throws Exception {
- StreamExecutionEnvironment env = new TestStreamEnvironment(SOURCE_PARALELISM, MEMORYSIZE);
-
- DataStream<String> fromStringElements = env.fromElements("aa", "bb", "cc");
- DataStream<Long> generatedSequence = env.generateSequence(0, 3);
-
- fromStringElements.connect(generatedSequence).map(new CoMap()).addSink(new SetSink());
-
- resultSet = new HashSet<String>();
- env.execute();
-
- HashSet<String> expectedSet = new HashSet<String>(Arrays.asList("aa", "bb", "cc", "0", "1",
- "2", "3"));
- assertEquals(expectedSet, resultSet);
- }
-
- @Test
- public void runStream() throws Exception {
- StreamExecutionEnvironment env = new TestStreamEnvironment(SOURCE_PARALELISM, MEMORYSIZE);
-
- env.addSource(new MySource()).setParallelism(SOURCE_PARALELISM).map(new MyTask()).addSink(new MySink());
-
- env.execute();
- assertEquals(10, data.keySet().size());
-
- for (Integer k : data.keySet()) {
- assertEquals((Integer) (k + 1), data.get(k));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
deleted file mode 100644
index e12b254..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.deltafunction;
-
-import static org.junit.Assert.*;
-
-import org.junit.Test;
-
-public class CosineDistanceTest {
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- @Test
- public void testCosineDistance() {
-
- //Reference calculated using wolfram alpha
- double[][][] testdata={
- {{0,0,0},{0,0,0}},
- {{0,0,0},{1,2,3}},
- {{1,2,3},{0,0,0}},
- {{1,2,3},{4,5,6}},
- {{1,2,3},{-4,-5,-6}},
- {{1,2,-3},{-4,5,-6}},
- {{1,2,3,4},{5,6,7,8}},
- {{1,2},{3,4}},
- {{1},{2}},
- };
- double[] referenceSolutions={
- 0,
- 0,
- 0,
- 0.025368,
- 1.974631,
- 0.269026,
- 0.031136,
- 0.016130,
- 0
- };
-
- for (int i = 0; i < testdata.length; i++) {
- assertEquals("Wrong result for inputs " + arrayToString(testdata[i][0]) + " and "
- + arrayToString(testdata[i][0]), referenceSolutions[i],
- new CosineDistance().getDelta(testdata[i][0], testdata[i][1]), 0.000001);
- }
- }
-
- private String arrayToString(double[] in){
- if (in.length==0) return "{}";
- String result="{";
- for (double d:in){
- result+=d+",";
- }
- return result.substring(0, result.length()-1)+"}";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
deleted file mode 100644
index 8c62497..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.deltafunction;
-
-import static org.junit.Assert.*;
-
-import org.junit.Test;
-
-public class EuclideanDistanceTest {
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- @Test
- public void testEuclideanDistance() {
-
- //Reference calculated using wolfram alpha
- double[][][] testdata={
- {{0,0,0},{0,0,0}},
- {{0,0,0},{1,2,3}},
- {{1,2,3},{0,0,0}},
- {{1,2,3},{4,5,6}},
- {{1,2,3},{-4,-5,-6}},
- {{1,2,-3},{-4,5,-6}},
- {{1,2,3,4},{5,6,7,8}},
- {{1,2},{3,4}},
- {{1},{2}},
- };
- double[] referenceSolutions={
- 0,
- 3.741657,
- 3.741657,
- 5.196152,
- 12.4499,
- 6.557439,
- 8.0,
- 2.828427,
- 1
- };
-
- for (int i = 0; i < testdata.length; i++) {
- assertEquals("Wrong result for inputs " + arrayToString(testdata[i][0]) + " and "
- + arrayToString(testdata[i][0]), referenceSolutions[i],
- new EuclideanDistance().getDelta(testdata[i][0], testdata[i][1]), 0.000001);
- }
-
- }
-
- private String arrayToString(double[] in){
- if (in.length==0) return "{}";
- String result="{";
- for (double d:in){
- result+=d+",";
- }
- return result.substring(0, result.length()-1)+"}";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java
deleted file mode 100644
index 17d3974..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple10;
-import org.apache.flink.api.java.tuple.Tuple11;
-import org.apache.flink.api.java.tuple.Tuple12;
-import org.apache.flink.api.java.tuple.Tuple13;
-import org.apache.flink.api.java.tuple.Tuple14;
-import org.apache.flink.api.java.tuple.Tuple15;
-import org.apache.flink.api.java.tuple.Tuple16;
-import org.apache.flink.api.java.tuple.Tuple17;
-import org.apache.flink.api.java.tuple.Tuple18;
-import org.apache.flink.api.java.tuple.Tuple19;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple20;
-import org.apache.flink.api.java.tuple.Tuple21;
-import org.apache.flink.api.java.tuple.Tuple22;
-import org.apache.flink.api.java.tuple.Tuple23;
-import org.apache.flink.api.java.tuple.Tuple24;
-import org.apache.flink.api.java.tuple.Tuple25;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.api.java.tuple.Tuple6;
-import org.apache.flink.api.java.tuple.Tuple7;
-import org.apache.flink.api.java.tuple.Tuple8;
-import org.apache.flink.api.java.tuple.Tuple9;
-import org.apache.flink.streaming.api.windowing.extractor.ArrayFromTuple;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ArrayFromTupleTest {
-
- private String[] testStrings;
-
- @Before
- public void init() {
- testStrings = new String[Tuple.MAX_ARITY];
- for (int i = 0; i < Tuple.MAX_ARITY; i++) {
- testStrings[i] = Integer.toString(i);
- }
- }
-
- @Test
- public void testConvertFromTupleToArray() throws InstantiationException, IllegalAccessException {
- for (int i = 0; i < Tuple.MAX_ARITY; i++) {
- Tuple currentTuple = (Tuple) CLASSES[i].newInstance();
- String[] currentArray = new String[i + 1];
- for (int j = 0; j <= i; j++) {
- currentTuple.setField(testStrings[j], j);
- currentArray[j] = testStrings[j];
- }
- arrayEqualityCheck(currentArray, new ArrayFromTuple().extract(currentTuple));
- }
- }
-
- @Test
- public void testUserSpecifiedOrder() throws InstantiationException, IllegalAccessException {
- Tuple currentTuple = (Tuple) CLASSES[Tuple.MAX_ARITY - 1].newInstance();
- for (int i = 0; i < Tuple.MAX_ARITY; i++) {
- currentTuple.setField(testStrings[i], i);
- }
-
- String[] expected = { testStrings[5], testStrings[3], testStrings[6], testStrings[7],
- testStrings[0] };
- arrayEqualityCheck(expected, new ArrayFromTuple(5, 3, 6, 7, 0).extract(currentTuple));
-
- String[] expected2 = { testStrings[0], testStrings[Tuple.MAX_ARITY - 1] };
- arrayEqualityCheck(expected2,
- new ArrayFromTuple(0, Tuple.MAX_ARITY - 1).extract(currentTuple));
-
- String[] expected3 = { testStrings[Tuple.MAX_ARITY - 1], testStrings[0] };
- arrayEqualityCheck(expected3,
- new ArrayFromTuple(Tuple.MAX_ARITY - 1, 0).extract(currentTuple));
-
- String[] expected4 = { testStrings[13], testStrings[4], testStrings[5], testStrings[4],
- testStrings[2], testStrings[8], testStrings[6], testStrings[2], testStrings[8],
- testStrings[3], testStrings[5], testStrings[2], testStrings[16], testStrings[4],
- testStrings[3], testStrings[2], testStrings[6], testStrings[4], testStrings[7],
- testStrings[4], testStrings[2], testStrings[8], testStrings[7], testStrings[2] };
- arrayEqualityCheck(expected4, new ArrayFromTuple(13, 4, 5, 4, 2, 8, 6, 2, 8, 3, 5, 2, 16,
- 4, 3, 2, 6, 4, 7, 4, 2, 8, 7, 2).extract(currentTuple));
- }
-
- private void arrayEqualityCheck(Object[] array1, Object[] array2) {
- assertEquals("The result arrays must have the same length", array1.length, array2.length);
- for (int i = 0; i < array1.length; i++) {
- assertEquals("Unequal fields at position " + i, array1[i], array2[i]);
- }
- }
-
- private static final Class<?>[] CLASSES = new Class<?>[] { Tuple1.class, Tuple2.class,
- Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
- Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
- Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
- Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
- Tuple24.class, Tuple25.class };
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java
deleted file mode 100644
index 82a876a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.windowing.extractor.ArrayFromTuple;
-import org.apache.flink.streaming.api.windowing.extractor.ConcatinatedExtract;
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
-import org.apache.flink.streaming.api.windowing.extractor.FieldFromArray;
-import org.apache.flink.streaming.api.windowing.extractor.FieldFromTuple;
-import org.apache.flink.streaming.api.windowing.extractor.FieldsFromArray;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ConcatinatedExtractTest {
-
- private String[] testStringArray1 = { "1", "2", "3" };
- private int[] testIntArray1 = { 1, 2, 3 };
- private String[] testStringArray2 = { "4", "5", "6" };
- private int[] testIntArray2 = { 4, 5, 6 };
- private String[] testStringArray3 = { "7", "8", "9" };
- private int[] testIntArray3 = { 7, 8, 9 };
- private Tuple2<String[], int[]>[] testTuple2Array;
- private Tuple2<String[], int[]> testTuple2;
- private Tuple2<Tuple2<String[], int[]>, Tuple2<String[], int[]>[]> testData;
-
- @SuppressWarnings("unchecked")
- @Before
- public void setupData() {
- testTuple2Array = new Tuple2[2];
- testTuple2Array[0] = new Tuple2<String[], int[]>(testStringArray1, testIntArray2);
- testTuple2Array[1] = new Tuple2<String[], int[]>(testStringArray2, testIntArray1);
-
- testTuple2 = new Tuple2<String[], int[]>(testStringArray3, testIntArray3);
-
- testData = new Tuple2<Tuple2<String[], int[]>, Tuple2<String[], int[]>[]>(testTuple2,
- testTuple2Array);
- }
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- @Test
- public void test1() {
- Extractor ext = new ConcatinatedExtract(new FieldFromTuple(0), new FieldFromTuple(1))
- .add(new FieldsFromArray(Integer.class, 2, 1, 0));
- int[] expected = { testIntArray3[2], testIntArray3[1], testIntArray3[0] };
- assertEquals(new Integer(expected[0]), ((Integer[]) ext.extract(testData))[0]);
- assertEquals(new Integer(expected[1]), ((Integer[]) ext.extract(testData))[1]);
- assertEquals(new Integer(expected[2]), ((Integer[]) ext.extract(testData))[2]);
- }
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- @Test
- public void test2() {
- Extractor ext = new ConcatinatedExtract(new FieldFromTuple(1), // Tuple2<String[],int[]>[]
- new FieldsFromArray(Tuple2.class, 1)) // Tuple2<String[],int[]>[]
- .add(new FieldFromArray(0)) // Tuple2<String[],int[]>
- .add(new ArrayFromTuple(0)) // Object[] (Containing String[])
- .add(new FieldFromArray(0)) // String[]
- .add(new FieldFromArray(1)); // String
-
- String expected2 = testStringArray2[1];
- assertEquals(expected2, ext.extract(testData));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java
deleted file mode 100644
index 2d4dbcf..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.streaming.api.windowing.extractor.FieldFromArray;
-import org.junit.Test;
-
-public class FieldFromArrayTest {
-
- String[] testStringArray = { "0", "1", "2", "3", "4" };
- Integer[] testIntegerArray = { 10, 11, 12, 13, 14 };
- int[] testIntArray = { 20, 21, 22, 23, 24 };
-
- @Test
- public void testStringArray() {
- for (int i = 0; i < this.testStringArray.length; i++) {
- assertEquals(this.testStringArray[i],
- new FieldFromArray<String>(i).extract(testStringArray));
- }
- }
-
- @Test
- public void testIntegerArray() {
- for (int i = 0; i < this.testIntegerArray.length; i++) {
- assertEquals(this.testIntegerArray[i],
- new FieldFromArray<String>(i).extract(testIntegerArray));
- }
- }
-
- @Test
- public void testIntArray() {
- for (int i = 0; i < this.testIntArray.length; i++) {
- assertEquals(new Integer(this.testIntArray[i]),
- new FieldFromArray<Integer>(i).extract(testIntArray));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java
deleted file mode 100644
index 528611a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple10;
-import org.apache.flink.api.java.tuple.Tuple11;
-import org.apache.flink.api.java.tuple.Tuple12;
-import org.apache.flink.api.java.tuple.Tuple13;
-import org.apache.flink.api.java.tuple.Tuple14;
-import org.apache.flink.api.java.tuple.Tuple15;
-import org.apache.flink.api.java.tuple.Tuple16;
-import org.apache.flink.api.java.tuple.Tuple17;
-import org.apache.flink.api.java.tuple.Tuple18;
-import org.apache.flink.api.java.tuple.Tuple19;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple20;
-import org.apache.flink.api.java.tuple.Tuple21;
-import org.apache.flink.api.java.tuple.Tuple22;
-import org.apache.flink.api.java.tuple.Tuple23;
-import org.apache.flink.api.java.tuple.Tuple24;
-import org.apache.flink.api.java.tuple.Tuple25;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.api.java.tuple.Tuple6;
-import org.apache.flink.api.java.tuple.Tuple7;
-import org.apache.flink.api.java.tuple.Tuple8;
-import org.apache.flink.api.java.tuple.Tuple9;
-import org.apache.flink.streaming.api.windowing.extractor.FieldFromTuple;
-import org.junit.Before;
-import org.junit.Test;
-
-public class FieldFromTupleTest {
-
- private String[] testStrings;
-
- @Before
- public void init() {
- testStrings = new String[Tuple.MAX_ARITY];
- for (int i = 0; i < Tuple.MAX_ARITY; i++) {
- testStrings[i] = Integer.toString(i);
- }
- }
-
- @Test
- public void testSingleFieldExtraction() throws InstantiationException, IllegalAccessException {
- // extract single fields
- for (int i = 0; i < Tuple.MAX_ARITY; i++) {
- Tuple current = (Tuple) CLASSES[i].newInstance();
- for (int j = 0; j < i; j++) {
- current.setField(testStrings[j], j);
- }
- for (int j = 0; j < i; j++) {
- assertEquals(testStrings[j], new FieldFromTuple<String>(j).extract(current));
- }
- }
- }
-
- private static final Class<?>[] CLASSES = new Class<?>[] { Tuple1.class, Tuple2.class,
- Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
- Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
- Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
- Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
- Tuple24.class, Tuple25.class };
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java
deleted file mode 100644
index 3139aa5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.flink.streaming.api.windowing.extractor.FieldsFromArray;
-import org.junit.Test;
-
-public class FieldsFromArrayTest {
-
- String[] testStringArray = { "0", "1", "2", "3", "4" };
- Integer[] testIntegerArray = { 10, 11, 12, 13, 14 };
- int[] testIntArray = { 20, 21, 22, 23, 24 };
-
- @Test
- public void testStringArray() {
- // check single field extraction
- for (int i = 0; i < testStringArray.length; i++) {
- String[] tmp = { testStringArray[i] };
- arrayEqualityCheck(tmp,
- new FieldsFromArray<String>(String.class, i).extract(testStringArray));
- }
-
- // check reverse order
- String[] reverseOrder = new String[testStringArray.length];
- for (int i = 0; i < testStringArray.length; i++) {
- reverseOrder[i] = testStringArray[testStringArray.length - i - 1];
- }
- arrayEqualityCheck(reverseOrder,
- new FieldsFromArray<String>(String.class, 4, 3, 2, 1, 0).extract(testStringArray));
-
- // check picking fields and reorder
- String[] crazyOrder = { testStringArray[4], testStringArray[1], testStringArray[2] };
- arrayEqualityCheck(crazyOrder,
- new FieldsFromArray<String>(String.class, 4, 1, 2).extract(testStringArray));
- }
-
- @Test
- public void testIntegerArray() {
- // check single field extraction
- for (int i = 0; i < testIntegerArray.length; i++) {
- Integer[] tmp = { testIntegerArray[i] };
- arrayEqualityCheck(tmp,
- new FieldsFromArray<Integer>(Integer.class, i).extract(testIntegerArray));
- }
-
- // check reverse order
- Integer[] reverseOrder = new Integer[testIntegerArray.length];
- for (int i = 0; i < testIntegerArray.length; i++) {
- reverseOrder[i] = testIntegerArray[testIntegerArray.length - i - 1];
- }
- arrayEqualityCheck(reverseOrder,
- new FieldsFromArray<Integer>(Integer.class, 4, 3, 2, 1, 0)
- .extract(testIntegerArray));
-
- // check picking fields and reorder
- Integer[] crazyOrder = { testIntegerArray[4], testIntegerArray[1], testIntegerArray[2] };
- arrayEqualityCheck(crazyOrder,
- new FieldsFromArray<Integer>(Integer.class, 4, 1, 2).extract(testIntegerArray));
-
- }
-
- @Test
- public void testIntArray() {
- for (int i = 0; i < testIntArray.length; i++) {
- Integer[] tmp = { testIntArray[i] };
- arrayEqualityCheck(tmp,
- new FieldsFromArray<Integer>(Integer.class, i).extract(testIntArray));
- }
-
- // check reverse order
- Integer[] reverseOrder = new Integer[testIntArray.length];
- for (int i = 0; i < testIntArray.length; i++) {
- reverseOrder[i] = testIntArray[testIntArray.length - i - 1];
- }
- arrayEqualityCheck(reverseOrder,
- new FieldsFromArray<Integer>(Integer.class, 4, 3, 2, 1, 0).extract(testIntArray));
-
- // check picking fields and reorder
- Integer[] crazyOrder = { testIntArray[4], testIntArray[1], testIntArray[2] };
- arrayEqualityCheck(crazyOrder,
- new FieldsFromArray<Integer>(Integer.class, 4, 1, 2).extract(testIntArray));
-
- }
-
- private void arrayEqualityCheck(Object[] array1, Object[] array2) {
- assertEquals("The result arrays must have the same length", array1.length, array2.length);
- for (int i = 0; i < array1.length; i++) {
- assertEquals("Unequal fields at position " + i, array1[i], array2[i]);
- }
- }
-}
[48/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableComparableWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableComparableWrapper.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableComparableWrapper.java
deleted file mode 100644
index 0519ac9..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableComparableWrapper.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
-
-import org.apache.flink.types.Key;
-import org.apache.hadoop.io.WritableComparable;
-
-public class WritableComparableWrapper<T extends WritableComparable<T>> extends WritableWrapper<T> implements Key<WritableComparableWrapper<T>> {
- private static final long serialVersionUID = 1L;
-
- public WritableComparableWrapper() {
- super();
- }
-
- public WritableComparableWrapper(T toWrap) {
- super(toWrap);
- }
-
- @Override
- public int compareTo(WritableComparableWrapper<T> o) {
- return super.value().compareTo(o.value());
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapper.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapper.java
deleted file mode 100644
index 6369bb7..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapper.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
-
-import java.io.IOException;
-
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.types.Value;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.hadoop.io.Writable;
-
-public class WritableWrapper<T extends Writable> implements Value {
- private static final long serialVersionUID = 2L;
-
- private T wrapped;
- private String wrappedType;
- private ClassLoader cl;
-
- public WritableWrapper() {
- }
-
- public WritableWrapper(T toWrap) {
- wrapped = toWrap;
- wrappedType = toWrap.getClass().getCanonicalName();
- }
-
- public T value() {
- return wrapped;
- }
-
- @Override
- public void write(DataOutputView out) throws IOException {
- out.writeUTF(wrappedType);
- wrapped.write(out);
- }
-
- @Override
- public void read(DataInputView in) throws IOException {
- if(cl == null) {
- cl = Thread.currentThread().getContextClassLoader();
- }
- wrappedType = in.readUTF();
- try {
- @SuppressWarnings("unchecked")
- Class<T> wrClass = (Class<T>) Class.forName(wrappedType, true, cl).asSubclass(Writable.class);
- wrapped = InstantiationUtil.instantiate(wrClass, Writable.class);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException("Error creating the WritableWrapper", e);
- }
- wrapped.readFields(in);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapperConverter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapperConverter.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapperConverter.java
deleted file mode 100644
index 6bb13d1..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapperConverter.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
-
-import org.apache.flink.types.Record;
-import org.apache.flink.types.Value;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-@SuppressWarnings("rawtypes")
-public class WritableWrapperConverter<K extends WritableComparable, V extends Writable> implements HadoopTypeConverter<K,V> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void convert(Record flinkRecord, K hadoopKey, V hadoopValue) {
- flinkRecord.setField(0, convertKey(hadoopKey));
- flinkRecord.setField(1, convertValue(hadoopValue));
- }
-
- @SuppressWarnings("unchecked")
- private final Value convertKey(K in) {
- return new WritableComparableWrapper(in);
- }
-
- private final Value convertValue(V in) {
- return new WritableWrapper<V>(in);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java
deleted file mode 100644
index b167080..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapred.record.example;
-
-import java.io.Serializable;
-import java.util.Iterator;
-import java.util.StringTokenizer;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.Program;
-import org.apache.flink.api.common.ProgramDescription;
-import org.apache.flink.api.java.record.functions.MapFunction;
-import org.apache.flink.api.java.record.functions.ReduceFunction;
-import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFields;
-import org.apache.flink.api.java.record.io.CsvOutputFormat;
-import org.apache.flink.api.java.record.operators.FileDataSink;
-import org.apache.flink.api.java.record.operators.MapOperator;
-import org.apache.flink.api.java.record.operators.ReduceOperator;
-import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
-import org.apache.flink.client.LocalExecutor;
-import org.apache.flink.hadoopcompatibility.mapred.record.HadoopDataSource;
-import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.WritableWrapperConverter;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.StringValue;
-import org.apache.flink.util.Collector;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-/**
- * Implements a word count which takes the input file and counts the number of
- * the occurrences of each word in the file.
- *
- * <br /><br />
- *
- * <b>Note</b>: This example uses the out-dated Record API.
- * It is recommended to use the new Java API.
- *
- * @see org.apache.flink.hadoopcompatibility.mapred.example.HadoopMapredCompatWordCount
- */
-public class WordCount implements Program, ProgramDescription {
-
- private static final long serialVersionUID = 1L;
-
-
- /**
- * Converts a Record containing one string in to multiple string/integer pairs.
- * The string is tokenized by whitespaces. For each token a new record is emitted,
- * where the token is the first field and an Integer(1) is the second field.
- */
- public static class TokenizeLine extends MapFunction implements Serializable {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void map(Record record, Collector<Record> collector) {
- // get the first field (as type StringValue) from the record
- String line = record.getField(1, StringValue.class).getValue();
- // normalize the line
- line = line.replaceAll("\\W+", " ").toLowerCase();
-
- // tokenize the line
- StringTokenizer tokenizer = new StringTokenizer(line);
- while (tokenizer.hasMoreTokens()) {
- String word = tokenizer.nextToken();
-
- // we emit a (word, 1) pair
- collector.collect(new Record(new StringValue(word), new IntValue(1)));
- }
- }
- }
-
- /**
- * Sums up the counts for a certain given key. The counts are assumed to be at position <code>1</code>
- * in the record. The other fields are not modified.
- */
- @Combinable
- @ConstantFields(0)
- public static class CountWords extends ReduceFunction implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
- Record element = null;
- int sum = 0;
-
- while (records.hasNext()) {
- element = records.next();
- int cnt = element.getField(1, IntValue.class).getValue();
- sum += cnt;
- }
-
- element.setField(1, new IntValue(sum));
- out.collect(element);
- }
-
- @Override
- public void combine(Iterator<Record> records, Collector<Record> out) throws Exception {
- // the logic is the same as in the reduce function, so simply call the reduce method
- reduce(records, out);
- }
- }
-
-
- @SuppressWarnings({ "rawtypes", "unchecked", "unused" })
- @Override
- public Plan getPlan(String... args) {
- // parse job parameters
- int numSubTasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
- String dataInput = (args.length > 1 ? args[1] : "");
- String output = (args.length > 2 ? args[2] : "");
-
-
- HadoopDataSource source = new HadoopDataSource(new TextInputFormat(), new JobConf(), "Input Lines");
- TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
-
- // Example with Wrapper Converter
- HadoopDataSource<LongWritable,Text> sourceHadoopType = new HadoopDataSource<LongWritable, Text>(
- new TextInputFormat(), new JobConf(), "Input Lines", new WritableWrapperConverter<LongWritable, Text>());
- TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
-
- MapOperator mapper = MapOperator.builder(new TokenizeLine())
- .input(source)
- .name("Tokenize Lines")
- .build();
- ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
- .input(mapper)
- .name("Count Words")
- .build();
- FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, reducer, "Word Counts");
- CsvOutputFormat.configureRecordFormat(out)
- .recordDelimiter('\n')
- .fieldDelimiter(' ')
- .field(StringValue.class, 0)
- .field(IntValue.class, 1);
-
- Plan plan = new Plan(out, "WordCount Example");
- plan.setDefaultParallelism(numSubTasks);
- return plan;
- }
-
-
- @Override
- public String getDescription() {
- return "Parameters: [numSubStasks] [input] [output]";
- }
-
-
- public static void main(String[] args) throws Exception {
- WordCount wc = new WordCount();
-
- if (args.length < 3) {
- System.err.println(wc.getDescription());
- System.exit(1);
- }
-
- Plan plan = wc.getPlan(args);
-
- // This will execute the word-count embedded in a local context. replace this line by the commented
- // succeeding line to send the job to a local installation or to a cluster for execution
- LocalExecutor.execute(plan);
-// PlanExecutor ex = new RemoteExecutor("localhost", 6123, "target/pact-examples-0.4-SNAPSHOT-WordCount.jar");
-// ex.executePlan(plan);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java
deleted file mode 100644
index a838215..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapred.record.example;
-
-import java.io.Serializable;
-import java.util.Iterator;
-import java.util.StringTokenizer;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.Program;
-import org.apache.flink.api.common.ProgramDescription;
-import org.apache.flink.api.java.record.functions.MapFunction;
-import org.apache.flink.api.java.record.functions.ReduceFunction;
-import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFields;
-import org.apache.flink.api.java.record.operators.MapOperator;
-import org.apache.flink.api.java.record.operators.ReduceOperator;
-import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
-import org.apache.flink.client.LocalExecutor;
-import org.apache.flink.hadoopcompatibility.mapred.record.HadoopDataSink;
-import org.apache.flink.hadoopcompatibility.mapred.record.HadoopDataSource;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.StringValue;
-import org.apache.flink.util.Collector;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TextInputFormat;
-import org.apache.hadoop.mapred.TextOutputFormat;
-
-/**
- * Implements a word count which takes the input file and counts the number of
- * the occurrences of each word in the file.
- *
- * <br /><br />
- *
- * <b>Note</b>: This example uses the out dated Record API.
- * It is recommended to use the new Java API.
- *
- * @see WordCount
- */
-@SuppressWarnings("serial")
-public class WordCountWithOutputFormat implements Program, ProgramDescription {
-
- /**
- * Converts a Record containing one string in to multiple string/integer pairs.
- * The string is tokenized by whitespaces. For each token a new record is emitted,
- * where the token is the first field and an Integer(1) is the second field.
- */
- public static class TokenizeLine extends MapFunction implements Serializable {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void map(Record record, Collector<Record> collector) {
- // get the first field (as type StringValue) from the record
- String line = record.getField(1, StringValue.class).getValue();
- // normalize the line
- line = line.replaceAll("\\W+", " ").toLowerCase();
-
- // tokenize the line
- StringTokenizer tokenizer = new StringTokenizer(line);
- while (tokenizer.hasMoreTokens()) {
- String word = tokenizer.nextToken();
-
- // we emit a (word, 1) pair
- collector.collect(new Record(new StringValue(word), new IntValue(1)));
- }
- }
- }
-
- /**
- * Sums up the counts for a certain given key. The counts are assumed to be at position <code>1</code>
- * in the record. The other fields are not modified.
- */
- @Combinable
- @ConstantFields(0)
- public static class CountWords extends ReduceFunction implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
- Record element = null;
- int sum = 0;
-
- while (records.hasNext()) {
- element = records.next();
- int cnt = element.getField(1, IntValue.class).getValue();
- sum += cnt;
- }
-
- element.setField(1, new IntValue(sum));
- out.collect(element);
- }
-
- @Override
- public void combine(Iterator<Record> records, Collector<Record> out) throws Exception {
- // the logic is the same as in the reduce function, so simply call the reduce method
- reduce(records, out);
- }
- }
-
-
- @Override
- public Plan getPlan(String... args) {
- // parse job parameters
- int numSubTasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
- String dataInput = (args.length > 1 ? args[1] : "");
- String output = (args.length > 2 ? args[2] : "");
-
- HadoopDataSource<LongWritable, Text> source = new HadoopDataSource<LongWritable, Text>(
- new TextInputFormat(), new JobConf(), "Input Lines");
- TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
-
-
- MapOperator mapper = MapOperator.builder(new TokenizeLine())
- .input(source)
- .name("Tokenize Lines")
- .build();
- ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
- .input(mapper)
- .name("Count Words")
- .build();
- HadoopDataSink<Text, IntWritable> out = new HadoopDataSink<Text, IntWritable>(new TextOutputFormat<Text, IntWritable>(),new JobConf(), "Hadoop TextOutputFormat", reducer, Text.class, IntWritable.class);
- TextOutputFormat.setOutputPath(out.getJobConf(), new Path(output));
-
- Plan plan = new Plan(out, "Hadoop OutputFormat Example");
- plan.setDefaultParallelism(numSubTasks);
- return plan;
- }
-
-
- @Override
- public String getDescription() {
- return "Parameters: [numSubStasks] [input] [output]";
- }
-
-
- public static void main(String[] args) throws Exception {
- WordCountWithOutputFormat wc = new WordCountWithOutputFormat();
-
- if (args.length < 3) {
- System.err.println(wc.getDescription());
- System.exit(1);
- }
-
- Plan plan = wc.getPlan(args);
-
- // This will execute the word-count embedded in a local context. replace this line by the commented
- // succeeding line to send the job to a local installation or to a cluster for execution
- LocalExecutor.execute(plan);
-// PlanExecutor ex = new RemoteExecutor("localhost", 6123, "target/pact-examples-0.4-SNAPSHOT-WordCount.jar");
-// ex.executePlan(plan);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/utils/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/utils/HadoopUtils.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/utils/HadoopUtils.java
deleted file mode 100644
index 2d2f518..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/utils/HadoopUtils.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.utils;
-
-import java.lang.reflect.Constructor;
-import java.util.Map;
-
-import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
-import org.apache.hadoop.mapred.JobID;
-import org.apache.hadoop.mapred.TaskAttemptContext;
-import org.apache.hadoop.mapred.TaskAttemptID;
-
-
-public class HadoopUtils {
-
- /**
- * Merge HadoopConfiguration into JobConf. This is necessary for the HDFS configuration.
- */
- public static void mergeHadoopConf(JobConf jobConf) {
- org.apache.hadoop.conf.Configuration hadoopConf = HadoopFileSystem.getHadoopConfiguration();
- for (Map.Entry<String, String> e : hadoopConf) {
- jobConf.set(e.getKey(), e.getValue());
- }
- }
-
- public static JobContext instantiateJobContext(JobConf jobConf, JobID jobId) throws Exception {
- try {
- // for Hadoop 1.xx
- Class<?> clazz = null;
- if(!TaskAttemptContext.class.isInterface()) {
- clazz = Class.forName("org.apache.hadoop.mapred.JobContext", true, Thread.currentThread().getContextClassLoader());
- }
- // for Hadoop 2.xx
- else {
- clazz = Class.forName("org.apache.hadoop.mapred.JobContextImpl", true, Thread.currentThread().getContextClassLoader());
- }
- Constructor<?> constructor = clazz.getDeclaredConstructor(JobConf.class, org.apache.hadoop.mapreduce.JobID.class);
- // for Hadoop 1.xx
- constructor.setAccessible(true);
- JobContext context = (JobContext) constructor.newInstance(jobConf, jobId);
-
- return context;
- } catch(Exception e) {
- throw new Exception("Could not create instance of JobContext.", e);
- }
- }
-
- public static TaskAttemptContext instantiateTaskAttemptContext(JobConf jobConf, TaskAttemptID taskAttemptID) throws Exception {
- try {
- // for Hadoop 1.xx
- Class<?> clazz = null;
- if(!TaskAttemptContext.class.isInterface()) {
- clazz = Class.forName("org.apache.hadoop.mapred.TaskAttemptContext", true, Thread.currentThread().getContextClassLoader());
- }
- // for Hadoop 2.xx
- else {
- clazz = Class.forName("org.apache.hadoop.mapred.TaskAttemptContextImpl", true, Thread.currentThread().getContextClassLoader());
- }
- Constructor<?> constructor = clazz.getDeclaredConstructor(JobConf.class, TaskAttemptID.class);
- // for Hadoop 1.xx
- constructor.setAccessible(true);
- TaskAttemptContext context = (TaskAttemptContext) constructor.newInstance(jobConf, taskAttemptID);
- return context;
- } catch(Exception e) {
- throw new Exception("Could not create instance of TaskAttemptContext.", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyProgressable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyProgressable.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyProgressable.java
deleted file mode 100644
index 483dd2f..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyProgressable.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.wrapper;
-
-import org.apache.hadoop.util.Progressable;
-
-/**
- * This is a dummy progress
- *
- */
-public class HadoopDummyProgressable implements Progressable {
- @Override
- public void progress() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
deleted file mode 100644
index 84a1e9e..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.wrapper;
-
-import org.apache.hadoop.mapred.Counters.Counter;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.Reporter;
-
-/**
- * This is a dummy progress monitor / reporter
- *
- */
-public class HadoopDummyReporter implements Reporter {
-
- @Override
- public void progress() {
- }
-
- @Override
- public void setStatus(String status) {
-
- }
-
- @Override
- public Counter getCounter(Enum<?> name) {
- return null;
- }
-
- @Override
- public Counter getCounter(String group, String name) {
- return null;
- }
-
- @Override
- public void incrCounter(Enum<?> key, long amount) {
-
- }
-
- @Override
- public void incrCounter(String group, String counter, long amount) {
-
- }
-
- @Override
- public InputSplit getInputSplit() throws UnsupportedOperationException {
- return null;
- }
- // There should be an @Override, but some CDH4 dependency does not contain this method
- public float getProgress() {
- return 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopInputSplit.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopInputSplit.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopInputSplit.java
deleted file mode 100644
index aa2155d..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopInputSplit.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.wrapper;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
-import org.apache.flink.core.io.LocatableInputSplit;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.mapred.JobConf;
-
-
-public class HadoopInputSplit extends LocatableInputSplit {
-
- private static final long serialVersionUID = 1L;
-
-
- private transient org.apache.hadoop.mapred.InputSplit hadoopInputSplit;
-
- private JobConf jobConf;
-
- private int splitNumber;
- private String hadoopInputSplitTypeName;
-
-
- public org.apache.hadoop.mapred.InputSplit getHadoopInputSplit() {
- return hadoopInputSplit;
- }
-
- public HadoopInputSplit() {
- super();
- }
-
- public HadoopInputSplit(int splitNumber, org.apache.hadoop.mapred.InputSplit hInputSplit, JobConf jobconf) {
-
- this.splitNumber = splitNumber;
- this.hadoopInputSplit = hInputSplit;
- this.hadoopInputSplitTypeName = hInputSplit.getClass().getName();
- this.jobConf = jobconf;
-
- }
-
- @Override
- public void write(DataOutputView out) throws IOException {
- out.writeInt(splitNumber);
- out.writeUTF(hadoopInputSplitTypeName);
- jobConf.write(out);
- hadoopInputSplit.write(out);
- }
-
- @Override
- public void read(DataInputView in) throws IOException {
- this.splitNumber = in.readInt();
- this.hadoopInputSplitTypeName = in.readUTF();
- if(hadoopInputSplit == null) {
- try {
- Class<? extends org.apache.hadoop.io.Writable> inputSplit =
- Class.forName(hadoopInputSplitTypeName).asSubclass(org.apache.hadoop.io.Writable.class);
- this.hadoopInputSplit = (org.apache.hadoop.mapred.InputSplit) WritableFactories.newInstance( inputSplit );
- }
- catch (Exception e) {
- throw new RuntimeException("Unable to create InputSplit", e);
- }
- }
- jobConf = new JobConf();
- jobConf.readFields(in);
- if (this.hadoopInputSplit instanceof Configurable) {
- ((Configurable) this.hadoopInputSplit).setConf(this.jobConf);
- }
- this.hadoopInputSplit.readFields(in);
-
- }
-
- private void writeObject(ObjectOutputStream out) throws IOException {
- out.writeInt(splitNumber);
- out.writeUTF(hadoopInputSplitTypeName);
- jobConf.write(out);
- hadoopInputSplit.write(out);
-
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
- this.splitNumber=in.readInt();
- this.hadoopInputSplitTypeName = in.readUTF();
- if(hadoopInputSplit == null) {
- try {
- Class<? extends org.apache.hadoop.io.Writable> inputSplit =
- Class.forName(hadoopInputSplitTypeName).asSubclass(org.apache.hadoop.io.Writable.class);
- this.hadoopInputSplit = (org.apache.hadoop.mapred.InputSplit) WritableFactories.newInstance( inputSplit );
- }
- catch (Exception e) {
- throw new RuntimeException("Unable to create InputSplit", e);
- }
- }
- jobConf = new JobConf();
- jobConf.readFields(in);
- if (this.hadoopInputSplit instanceof Configurable) {
- ((Configurable) this.hadoopInputSplit).setConf(this.jobConf);
- }
- this.hadoopInputSplit.readFields(in);
- }
-
- @Override
- public int getSplitNumber() {
- return this.splitNumber;
- }
-
- @Override
- public String[] getHostnames() {
- try {
- return this.hadoopInputSplit.getLocations();
- } catch(IOException ioe) {
- return new String[0];
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopOutputCollector.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopOutputCollector.java
deleted file mode 100644
index fcb6841..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopOutputCollector.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapred.wrapper;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.Collector;
-import org.apache.hadoop.mapred.OutputCollector;
-
-import java.io.IOException;
-
-/**
- * A Hadoop OutputCollector that wraps a Flink OutputCollector.
- * On each call of collect() the data is forwarded to the wrapped Flink collector.
- *
- */
-@SuppressWarnings("rawtypes")
-public final class HadoopOutputCollector<KEY,VALUE>
- implements OutputCollector<KEY,VALUE> {
-
- private Collector<Tuple2<KEY,VALUE>> flinkCollector;
-
- private final Tuple2<KEY,VALUE> outTuple = new Tuple2<KEY, VALUE>();
-
- /**
- * Set the wrapped Flink collector.
- *
- * @param flinkCollector The wrapped Flink OutputCollector.
- */
- public void setFlinkCollector(Collector<Tuple2<KEY, VALUE>> flinkCollector) {
- this.flinkCollector = flinkCollector;
- }
-
- /**
- * Use the wrapped Flink collector to collect a key-value pair for Flink.
- *
- * @param key the key to collect
- * @param val the value to collect
- * @throws IOException unexpected of key or value in key-value pair.
- */
- @Override
- public void collect(final KEY key, final VALUE val) throws IOException {
-
- this.outTuple.f0 = key;
- this.outTuple.f1 = val;
- this.flinkCollector.collect(outTuple);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIterator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIterator.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIterator.java
deleted file mode 100644
index 5ecac2e..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIterator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapred.wrapper;
-
-import java.util.Iterator;
-
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.operators.translation.TupleUnwrappingIterator;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-
-/**
- * Wraps a Flink Tuple2 (key-value-pair) iterator into an iterator over the second (value) field.
- */
-@SuppressWarnings("rawtypes")
-public class HadoopTupleUnwrappingIterator<KEY,VALUE>
- extends TupleUnwrappingIterator<VALUE, KEY> implements java.io.Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private Iterator<Tuple2<KEY,VALUE>> iterator;
-
- private final TypeSerializer<KEY> keySerializer;
-
- private boolean atFirst = false;
- private KEY curKey = null;
- private VALUE firstValue = null;
-
- public HadoopTupleUnwrappingIterator(Class<KEY> keyClass) {
- this.keySerializer = TypeExtractor.getForClass((Class<KEY>) keyClass).createSerializer();
- }
-
- /**
- * Set the Flink iterator to wrap.
- *
- * @param iterator The Flink iterator to wrap.
- */
- @Override()
- public void set(final Iterator<Tuple2<KEY,VALUE>> iterator) {
- this.iterator = iterator;
- if(this.hasNext()) {
- final Tuple2<KEY, VALUE> tuple = iterator.next();
- this.curKey = keySerializer.copy(tuple.f0);
- this.firstValue = tuple.f1;
- this.atFirst = true;
- } else {
- this.atFirst = false;
- }
- }
-
- @Override
- public boolean hasNext() {
- if(this.atFirst) {
- return true;
- }
- return iterator.hasNext();
- }
-
- @Override
- public VALUE next() {
- if(this.atFirst) {
- this.atFirst = false;
- return firstValue;
- }
-
- final Tuple2<KEY, VALUE> tuple = iterator.next();
- return tuple.f1;
- }
-
- public KEY getCurrentKey() {
- return this.curKey;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopInputFormat.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopInputFormat.java
deleted file mode 100644
index 20006b8..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopInputFormat.java
+++ /dev/null
@@ -1,338 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapreduce;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.api.common.io.LocatableInputSplitAssigner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
-import org.apache.flink.api.common.io.statistics.BaseStatistics;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.fs.FileStatus;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.core.io.InputSplitAssigner;
-import org.apache.flink.hadoopcompatibility.mapreduce.utils.HadoopUtils;
-import org.apache.flink.hadoopcompatibility.mapreduce.wrapper.HadoopInputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-
-public class HadoopInputFormat<K, V> implements InputFormat<Tuple2<K,V>, HadoopInputSplit>, ResultTypeQueryable<Tuple2<K,V>> {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(HadoopInputFormat.class);
-
- private org.apache.hadoop.mapreduce.InputFormat<K, V> mapreduceInputFormat;
- private Class<K> keyClass;
- private Class<V> valueClass;
- private org.apache.hadoop.conf.Configuration configuration;
-
- private transient RecordReader<K, V> recordReader;
- private boolean fetched = false;
- private boolean hasNext;
-
- public HadoopInputFormat() {
- super();
- }
-
- public HadoopInputFormat(org.apache.hadoop.mapreduce.InputFormat<K,V> mapreduceInputFormat, Class<K> key, Class<V> value, Job job) {
- super();
- this.mapreduceInputFormat = mapreduceInputFormat;
- this.keyClass = key;
- this.valueClass = value;
- this.configuration = job.getConfiguration();
- HadoopUtils.mergeHadoopConf(configuration);
- }
-
- public void setConfiguration(org.apache.hadoop.conf.Configuration configuration) {
- this.configuration = configuration;
- }
-
- public org.apache.hadoop.mapreduce.InputFormat<K,V> getHadoopInputFormat() {
- return this.mapreduceInputFormat;
- }
-
- public void setHadoopInputFormat(org.apache.hadoop.mapreduce.InputFormat<K,V> mapreduceInputFormat) {
- this.mapreduceInputFormat = mapreduceInputFormat;
- }
-
- public org.apache.hadoop.conf.Configuration getConfiguration() {
- return this.configuration;
- }
-
- // --------------------------------------------------------------------------------------------
- // InputFormat
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void configure(Configuration parameters) {
- // nothing to do
- }
-
- @Override
- public BaseStatistics getStatistics(BaseStatistics cachedStats) throws IOException {
- // only gather base statistics for FileInputFormats
- if(!(mapreduceInputFormat instanceof FileInputFormat)) {
- return null;
- }
-
- JobContext jobContext = null;
- try {
- jobContext = HadoopUtils.instantiateJobContext(configuration, null);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- final FileBaseStatistics cachedFileStats = (cachedStats != null && cachedStats instanceof FileBaseStatistics) ?
- (FileBaseStatistics) cachedStats : null;
-
- try {
- final org.apache.hadoop.fs.Path[] paths = FileInputFormat.getInputPaths(jobContext);
- return getFileStats(cachedFileStats, paths, new ArrayList<FileStatus>(1));
- } catch (IOException ioex) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Could not determine statistics due to an io error: "
- + ioex.getMessage());
- }
- } catch (Throwable t) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Unexpected problem while getting the file statistics: "
- + t.getMessage(), t);
- }
- }
-
- // no statistics available
- return null;
- }
-
- @Override
- public HadoopInputSplit[] createInputSplits(int minNumSplits)
- throws IOException {
- configuration.setInt("mapreduce.input.fileinputformat.split.minsize", minNumSplits);
-
- JobContext jobContext = null;
- try {
- jobContext = HadoopUtils.instantiateJobContext(configuration, new JobID());
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- List<org.apache.hadoop.mapreduce.InputSplit> splits;
- try {
- splits = this.mapreduceInputFormat.getSplits(jobContext);
- } catch (InterruptedException e) {
- throw new IOException("Could not get Splits.", e);
- }
- HadoopInputSplit[] hadoopInputSplits = new HadoopInputSplit[splits.size()];
-
- for(int i = 0; i < hadoopInputSplits.length; i++){
- hadoopInputSplits[i] = new HadoopInputSplit(i, splits.get(i), jobContext);
- }
- return hadoopInputSplits;
- }
-
- @Override
- public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
- return new LocatableInputSplitAssigner(inputSplits);
- }
-
- @Override
- public void open(HadoopInputSplit split) throws IOException {
- TaskAttemptContext context = null;
- try {
- context = HadoopUtils.instantiateTaskAttemptContext(configuration, new TaskAttemptID());
- } catch(Exception e) {
- throw new RuntimeException(e);
- }
-
- try {
- this.recordReader = this.mapreduceInputFormat
- .createRecordReader(split.getHadoopInputSplit(), context);
- this.recordReader.initialize(split.getHadoopInputSplit(), context);
- } catch (InterruptedException e) {
- throw new IOException("Could not create RecordReader.", e);
- } finally {
- this.fetched = false;
- }
- }
-
- @Override
- public boolean reachedEnd() throws IOException {
- if(!this.fetched) {
- fetchNext();
- }
- return !this.hasNext;
- }
-
- private void fetchNext() throws IOException {
- try {
- this.hasNext = this.recordReader.nextKeyValue();
- } catch (InterruptedException e) {
- throw new IOException("Could not fetch next KeyValue pair.", e);
- } finally {
- this.fetched = true;
- }
- }
-
- @Override
- public Tuple2<K, V> nextRecord(Tuple2<K, V> record) throws IOException {
- if(!this.fetched) {
- fetchNext();
- }
- if(!this.hasNext) {
- return null;
- }
- try {
- record.f0 = this.recordReader.getCurrentKey();
- record.f1 = this.recordReader.getCurrentValue();
- } catch (InterruptedException e) {
- throw new IOException("Could not get KeyValue pair.", e);
- }
- this.fetched = false;
-
- return record;
- }
-
- @Override
- public void close() throws IOException {
- this.recordReader.close();
- }
-
- // --------------------------------------------------------------------------------------------
- // Helper methods
- // --------------------------------------------------------------------------------------------
-
- private FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, org.apache.hadoop.fs.Path[] hadoopFilePaths,
- ArrayList<FileStatus> files) throws IOException {
-
- long latestModTime = 0L;
-
- // get the file info and check whether the cached statistics are still valid.
- for(org.apache.hadoop.fs.Path hadoopPath : hadoopFilePaths) {
-
- final Path filePath = new Path(hadoopPath.toUri());
- final FileSystem fs = FileSystem.get(filePath.toUri());
-
- final FileStatus file = fs.getFileStatus(filePath);
- latestModTime = Math.max(latestModTime, file.getModificationTime());
-
- // enumerate all files and check their modification time stamp.
- if (file.isDir()) {
- FileStatus[] fss = fs.listStatus(filePath);
- files.ensureCapacity(files.size() + fss.length);
-
- for (FileStatus s : fss) {
- if (!s.isDir()) {
- files.add(s);
- latestModTime = Math.max(s.getModificationTime(), latestModTime);
- }
- }
- } else {
- files.add(file);
- }
- }
-
- // check whether the cached statistics are still valid, if we have any
- if (cachedStats != null && latestModTime <= cachedStats.getLastModificationTime()) {
- return cachedStats;
- }
-
- // calculate the whole length
- long len = 0;
- for (FileStatus s : files) {
- len += s.getLen();
- }
-
- // sanity check
- if (len <= 0) {
- len = BaseStatistics.SIZE_UNKNOWN;
- }
-
- return new FileBaseStatistics(latestModTime, len, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
- }
-
- // --------------------------------------------------------------------------------------------
- // Custom serialization methods
- // --------------------------------------------------------------------------------------------
-
- private void writeObject(ObjectOutputStream out) throws IOException {
- out.writeUTF(this.mapreduceInputFormat.getClass().getName());
- out.writeUTF(this.keyClass.getName());
- out.writeUTF(this.valueClass.getName());
- this.configuration.write(out);
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
- String hadoopInputFormatClassName = in.readUTF();
- String keyClassName = in.readUTF();
- String valueClassName = in.readUTF();
-
- org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
- configuration.readFields(in);
-
- if(this.configuration == null) {
- this.configuration = configuration;
- }
-
- try {
- this.mapreduceInputFormat = (org.apache.hadoop.mapreduce.InputFormat<K,V>) Class.forName(hadoopInputFormatClassName, true, Thread.currentThread().getContextClassLoader()).newInstance();
- } catch (Exception e) {
- throw new RuntimeException("Unable to instantiate the hadoop input format", e);
- }
- try {
- this.keyClass = (Class<K>) Class.forName(keyClassName, true, Thread.currentThread().getContextClassLoader());
- } catch (Exception e) {
- throw new RuntimeException("Unable to find key class.", e);
- }
- try {
- this.valueClass = (Class<V>) Class.forName(valueClassName, true, Thread.currentThread().getContextClassLoader());
- } catch (Exception e) {
- throw new RuntimeException("Unable to find value class.", e);
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // ResultTypeQueryable
- // --------------------------------------------------------------------------------------------
-
- @Override
- public TypeInformation<Tuple2<K,V>> getProducedType() {
- return new TupleTypeInfo<Tuple2<K,V>>(TypeExtractor.createTypeInfo(keyClass), TypeExtractor.createTypeInfo(valueClass));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopOutputFormat.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopOutputFormat.java
deleted file mode 100644
index 696e1be..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopOutputFormat.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapreduce;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
-import org.apache.flink.api.common.io.FinalizeOnMaster;
-import org.apache.flink.api.common.io.OutputFormat;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.hadoopcompatibility.mapreduce.utils.HadoopUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
-
-
-public class HadoopOutputFormat<K,V> implements OutputFormat<Tuple2<K,V>>, FinalizeOnMaster {
-
- private static final long serialVersionUID = 1L;
-
- private org.apache.hadoop.conf.Configuration configuration;
- private org.apache.hadoop.mapreduce.OutputFormat<K,V> mapreduceOutputFormat;
- private transient RecordWriter<K,V> recordWriter;
- private transient FileOutputCommitter fileOutputCommitter;
- private transient TaskAttemptContext context;
- private transient int taskNumber;
-
- public HadoopOutputFormat(org.apache.hadoop.mapreduce.OutputFormat<K,V> mapreduceOutputFormat, Job job) {
- super();
- this.mapreduceOutputFormat = mapreduceOutputFormat;
- this.configuration = job.getConfiguration();
- HadoopUtils.mergeHadoopConf(configuration);
- }
-
- public void setConfiguration(org.apache.hadoop.conf.Configuration configuration) {
- this.configuration = configuration;
- }
-
- public org.apache.hadoop.conf.Configuration getConfiguration() {
- return this.configuration;
- }
-
- public org.apache.hadoop.mapreduce.OutputFormat<K,V> getHadoopOutputFormat() {
- return this.mapreduceOutputFormat;
- }
-
- public void setHadoopOutputFormat(org.apache.hadoop.mapreduce.OutputFormat<K,V> mapreduceOutputFormat) {
- this.mapreduceOutputFormat = mapreduceOutputFormat;
- }
-
- // --------------------------------------------------------------------------------------------
- // OutputFormat
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void configure(Configuration parameters) {
- // nothing to do
- }
-
- /**
- * create the temporary output file for hadoop RecordWriter.
- * @param taskNumber The number of the parallel instance.
- * @param numTasks The number of parallel tasks.
- * @throws IOException
- */
- @Override
- public void open(int taskNumber, int numTasks) throws IOException {
- if (Integer.toString(taskNumber + 1).length() > 6) {
- throw new IOException("Task id too large.");
- }
-
- this.taskNumber = taskNumber+1;
-
- // for hadoop 2.2
- this.configuration.set("mapreduce.output.basename", "tmp");
-
- TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_"
- + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s"," ").replace(" ", "0")
- + Integer.toString(taskNumber + 1)
- + "_0");
-
- this.configuration.set("mapred.task.id", taskAttemptID.toString());
- this.configuration.setInt("mapred.task.partition", taskNumber + 1);
- // for hadoop 2.2
- this.configuration.set("mapreduce.task.attempt.id", taskAttemptID.toString());
- this.configuration.setInt("mapreduce.task.partition", taskNumber + 1);
-
- try {
- this.context = HadoopUtils.instantiateTaskAttemptContext(this.configuration, taskAttemptID);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- this.fileOutputCommitter = new FileOutputCommitter(new Path(this.configuration.get("mapred.output.dir")), context);
-
- try {
- this.fileOutputCommitter.setupJob(HadoopUtils.instantiateJobContext(this.configuration, new JobID()));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- // compatible for hadoop 2.2.0, the temporary output directory is different from hadoop 1.2.1
- this.configuration.set("mapreduce.task.output.dir", this.fileOutputCommitter.getWorkPath().toString());
-
- try {
- this.recordWriter = this.mapreduceOutputFormat.getRecordWriter(this.context);
- } catch (InterruptedException e) {
- throw new IOException("Could not create RecordWriter.", e);
- }
- }
-
-
- @Override
- public void writeRecord(Tuple2<K, V> record) throws IOException {
- try {
- this.recordWriter.write(record.f0, record.f1);
- } catch (InterruptedException e) {
- throw new IOException("Could not write Record.", e);
- }
- }
-
- /**
- * commit the task by moving the output file out from the temporary directory.
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- try {
- this.recordWriter.close(this.context);
- } catch (InterruptedException e) {
- throw new IOException("Could not close RecordReader.", e);
- }
-
- if (this.fileOutputCommitter.needsTaskCommit(this.context)) {
- this.fileOutputCommitter.commitTask(this.context);
- }
-
- Path outputPath = new Path(this.configuration.get("mapred.output.dir"));
-
- // rename tmp-file to final name
- FileSystem fs = FileSystem.get(outputPath.toUri(), this.configuration);
-
- String taskNumberStr = Integer.toString(this.taskNumber);
- String tmpFileTemplate = "tmp-r-00000";
- String tmpFile = tmpFileTemplate.substring(0,11-taskNumberStr.length())+taskNumberStr;
-
- if(fs.exists(new Path(outputPath.toString()+"/"+tmpFile))) {
- fs.rename(new Path(outputPath.toString()+"/"+tmpFile), new Path(outputPath.toString()+"/"+taskNumberStr));
- }
- }
-
- @Override
- public void finalizeGlobal(int parallelism) throws IOException {
-
- JobContext jobContext;
- TaskAttemptContext taskContext;
- try {
-
- TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_"
- + String.format("%" + (6 - Integer.toString(1).length()) + "s"," ").replace(" ", "0")
- + Integer.toString(1)
- + "_0");
-
- jobContext = HadoopUtils.instantiateJobContext(this.configuration, new JobID());
- taskContext = HadoopUtils.instantiateTaskAttemptContext(this.configuration, taskAttemptID);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- this.fileOutputCommitter = new FileOutputCommitter(new Path(this.configuration.get("mapred.output.dir")), taskContext);
-
- // finalize HDFS output format
- this.fileOutputCommitter.commitJob(jobContext);
- }
-
- // --------------------------------------------------------------------------------------------
- // Custom serialization methods
- // --------------------------------------------------------------------------------------------
-
- private void writeObject(ObjectOutputStream out) throws IOException {
- out.writeUTF(this.mapreduceOutputFormat.getClass().getName());
- this.configuration.write(out);
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
- String hadoopOutputFormatClassName = in.readUTF();
-
- org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
- configuration.readFields(in);
-
- if(this.configuration == null) {
- this.configuration = configuration;
- }
-
- try {
- this.mapreduceOutputFormat = (org.apache.hadoop.mapreduce.OutputFormat<K,V>) Class.forName(hadoopOutputFormatClassName, true, Thread.currentThread().getContextClassLoader()).newInstance();
- } catch (Exception e) {
- throw new RuntimeException("Unable to instantiate the hadoop output format", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java
deleted file mode 100644
index 2b99fd2..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapreduce.example;
-
-import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.Collector;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.hadoopcompatibility.mapreduce.HadoopInputFormat;
-import org.apache.flink.hadoopcompatibility.mapreduce.HadoopOutputFormat;
-
-/**
- * Implements a word count which takes the input file and counts the number of
- * occurrences of each word in the file and writes the result back to disk.
- *
- * This example shows how to use Hadoop Input Formats, how to convert Hadoop Writables to
- * common Java types for better usage in a Flink job and how to use Hadoop Output Formats.
- */
-@SuppressWarnings("serial")
-public class WordCount {
-
- public static void main(String[] args) throws Exception {
- if (args.length < 2) {
- System.err.println("Usage: WordCount <input path> <result path>");
- return;
- }
-
- final String inputPath = args[0];
- final String outputPath = args[1];
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- // Set up the Hadoop Input Format
- Job job = Job.getInstance();
- HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, job);
- TextInputFormat.addInputPath(job, new Path(inputPath));
-
- // Create a Flink job with it
- DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);
-
- // Tokenize the line and convert from Writable "Text" to String for better handling
- DataSet<Tuple2<String, Integer>> words = text.flatMap(new Tokenizer());
-
- // Sum up the words
- DataSet<Tuple2<String, Integer>> result = words.groupBy(0).aggregate(Aggregations.SUM, 1);
-
- // Convert String back to Writable "Text" for use with Hadoop Output Format
- DataSet<Tuple2<Text, IntWritable>> hadoopResult = result.map(new HadoopDatatypeMapper());
-
- // Set up Hadoop Output Format
- HadoopOutputFormat<Text, IntWritable> hadoopOutputFormat = new HadoopOutputFormat<Text, IntWritable>(new TextOutputFormat<Text, IntWritable>(), job);
- hadoopOutputFormat.getConfiguration().set("mapreduce.output.textoutputformat.separator", " ");
- hadoopOutputFormat.getConfiguration().set("mapred.textoutputformat.separator", " "); // set the value for both, since this test
- // is being executed with both types (hadoop1 and hadoop2 profile)
- TextOutputFormat.setOutputPath(job, new Path(outputPath));
-
- // Output & Execute
- hadoopResult.output(hadoopOutputFormat);
- env.execute("Word Count");
- }
-
- /**
- * Splits a line into words and converts Hadoop Writables into normal Java data types.
- */
- public static final class Tokenizer extends RichFlatMapFunction<Tuple2<LongWritable, Text>, Tuple2<String, Integer>> {
-
- @Override
- public void flatMap(Tuple2<LongWritable, Text> value, Collector<Tuple2<String, Integer>> out) {
- // normalize and split the line
- String line = value.f1.toString();
- String[] tokens = line.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, 1));
- }
- }
- }
- }
-
- /**
- * Converts Java data types to Hadoop Writables.
- */
- public static final class HadoopDatatypeMapper extends RichMapFunction<Tuple2<String, Integer>, Tuple2<Text, IntWritable>> {
-
- @Override
- public Tuple2<Text, IntWritable> map(Tuple2<String, Integer> value) throws Exception {
- return new Tuple2<Text, IntWritable>(new Text(value.f0), new IntWritable(value.f1));
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/utils/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/utils/HadoopUtils.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/utils/HadoopUtils.java
deleted file mode 100644
index 86b730f..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/utils/HadoopUtils.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapreduce.utils;
-
-import java.lang.reflect.Constructor;
-import java.util.Map;
-
-import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-
-public class HadoopUtils {
-
- /**
- * Merge HadoopConfiguration into Configuration. This is necessary for the HDFS configuration.
- */
- public static void mergeHadoopConf(Configuration configuration) {
- Configuration hadoopConf = HadoopFileSystem.getHadoopConfiguration();
-
- for (Map.Entry<String, String> e : hadoopConf) {
- configuration.set(e.getKey(), e.getValue());
- }
- }
-
- public static JobContext instantiateJobContext(Configuration configuration, JobID jobId) throws Exception {
- try {
- Class<?> clazz;
- // for Hadoop 1.xx
- if(JobContext.class.isInterface()) {
- clazz = Class.forName("org.apache.hadoop.mapreduce.task.JobContextImpl", true, Thread.currentThread().getContextClassLoader());
- }
- // for Hadoop 2.xx
- else {
- clazz = Class.forName("org.apache.hadoop.mapreduce.JobContext", true, Thread.currentThread().getContextClassLoader());
- }
- Constructor<?> constructor = clazz.getConstructor(Configuration.class, JobID.class);
- JobContext context = (JobContext) constructor.newInstance(configuration, jobId);
-
- return context;
- } catch(Exception e) {
- throw new Exception("Could not create instance of JobContext.");
- }
- }
-
- public static TaskAttemptContext instantiateTaskAttemptContext(Configuration configuration, TaskAttemptID taskAttemptID) throws Exception {
- try {
- Class<?> clazz;
- // for Hadoop 1.xx
- if(JobContext.class.isInterface()) {
- clazz = Class.forName("org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl");
- }
- // for Hadoop 2.xx
- else {
- clazz = Class.forName("org.apache.hadoop.mapreduce.TaskAttemptContext");
- }
- Constructor<?> constructor = clazz.getConstructor(Configuration.class, TaskAttemptID.class);
- TaskAttemptContext context = (TaskAttemptContext) constructor.newInstance(configuration, taskAttemptID);
-
- return context;
- } catch(Exception e) {
- throw new Exception("Could not create instance of TaskAttemptContext.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/wrapper/HadoopInputSplit.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/wrapper/HadoopInputSplit.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/wrapper/HadoopInputSplit.java
deleted file mode 100644
index 7477c28..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/wrapper/HadoopInputSplit.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapreduce.wrapper;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
-import org.apache.flink.core.io.LocatableInputSplit;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.mapreduce.JobContext;
-
-
-public class HadoopInputSplit extends LocatableInputSplit {
-
- public transient org.apache.hadoop.mapreduce.InputSplit mapreduceInputSplit;
- public transient JobContext jobContext;
-
- private int splitNumber;
-
- public org.apache.hadoop.mapreduce.InputSplit getHadoopInputSplit() {
- return mapreduceInputSplit;
- }
-
-
- public HadoopInputSplit() {
- super();
- }
-
-
- public HadoopInputSplit(int splitNumber, org.apache.hadoop.mapreduce.InputSplit mapreduceInputSplit, JobContext jobContext) {
- this.splitNumber = splitNumber;
- if(!(mapreduceInputSplit instanceof Writable)) {
- throw new IllegalArgumentException("InputSplit must implement Writable interface.");
- }
- this.mapreduceInputSplit = mapreduceInputSplit;
- this.jobContext = jobContext;
- }
-
- @Override
- public void write(DataOutputView out) throws IOException {
- out.writeInt(this.splitNumber);
- out.writeUTF(this.mapreduceInputSplit.getClass().getName());
- Writable w = (Writable) this.mapreduceInputSplit;
- w.write(out);
- }
-
- @Override
- public void read(DataInputView in) throws IOException {
- this.splitNumber = in.readInt();
- String className = in.readUTF();
-
- if(this.mapreduceInputSplit == null) {
- try {
- Class<? extends org.apache.hadoop.io.Writable> inputSplit =
- Class.forName(className).asSubclass(org.apache.hadoop.io.Writable.class);
- this.mapreduceInputSplit = (org.apache.hadoop.mapreduce.InputSplit) WritableFactories.newInstance(inputSplit);
- } catch (Exception e) {
- throw new RuntimeException("Unable to create InputSplit", e);
- }
- }
- ((Writable)this.mapreduceInputSplit).readFields(in);
- }
-
- private void writeObject(ObjectOutputStream out) throws IOException {
- out.writeInt(this.splitNumber);
- out.writeUTF(this.mapreduceInputSplit.getClass().getName());
- Writable w = (Writable) this.mapreduceInputSplit;
- w.write(out);
-
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
- this.splitNumber=in.readInt();
- String className = in.readUTF();
-
- if(this.mapreduceInputSplit == null) {
- try {
- Class<? extends org.apache.hadoop.io.Writable> inputSplit =
- Class.forName(className).asSubclass(org.apache.hadoop.io.Writable.class);
- this.mapreduceInputSplit = (org.apache.hadoop.mapreduce.InputSplit) WritableFactories.newInstance(inputSplit);
- } catch (Exception e) {
- throw new RuntimeException("Unable to create InputSplit", e);
- }
- }
- ((Writable)this.mapreduceInputSplit).readFields(in);
- }
-
- @Override
- public int getSplitNumber() {
- return this.splitNumber;
- }
-
- @Override
- public String[] getHostnames() {
- try {
- return this.mapreduceInputSplit.getLocations();
- } catch (IOException e) {
- return new String[0];
- } catch (InterruptedException e) {
- return new String[0];
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopIOFormatsITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopIOFormatsITCase.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopIOFormatsITCase.java
deleted file mode 100644
index 32396b8..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopIOFormatsITCase.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.hadoopcompatibility.mapred;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.hadoopcompatibility.mapred.HadoopInputFormat;
-import org.apache.flink.test.util.JavaProgramTestBase;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.util.Collection;
-import java.util.LinkedList;
-
-@RunWith(Parameterized.class)
-public class HadoopIOFormatsITCase extends JavaProgramTestBase {
-
- private static int NUM_PROGRAMS = 2;
-
- private int curProgId = config.getInteger("ProgramId", -1);
- private String[] resultPath;
- private String[] expectedResult;
- private String sequenceFileInPath;
- private String sequenceFileInPathNull;
-
- public HadoopIOFormatsITCase(Configuration config) {
- super(config);
- }
-
- @Override
- protected void preSubmit() throws Exception {
- resultPath = new String[] {getTempDirPath("result0"), getTempDirPath("result1") };
-
- File sequenceFile = createAndRegisterTempFile("seqFile");
- sequenceFileInPath = sequenceFile.toURI().toString();
-
- // Create a sequence file
- org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
- FileSystem fs = FileSystem.get(URI.create(sequenceFile.getAbsolutePath()), conf);
- Path path = new Path(sequenceFile.getAbsolutePath());
-
- // ------------------ Long / Text Key Value pair: ------------
- int kvCount = 4;
-
- LongWritable key = new LongWritable();
- Text value = new Text();
- SequenceFile.Writer writer = null;
- try {
- writer = SequenceFile.createWriter( fs, conf, path, key.getClass(), value.getClass());
- for (int i = 0; i < kvCount; i ++) {
- if(i == 1) {
- // write key = 0 a bit more often.
- for(int a = 0;a < 15; a++) {
- key.set(i);
- value.set(i+" - somestring");
- writer.append(key, value);
- }
- }
- key.set(i);
- value.set(i+" - somestring");
- writer.append(key, value);
- }
- } finally {
- IOUtils.closeStream(writer);
- }
-
-
- // ------------------ Long / Text Key Value pair: ------------
-
- File sequenceFileNull = createAndRegisterTempFile("seqFileNullKey");
- sequenceFileInPathNull = sequenceFileNull.toURI().toString();
- path = new Path(sequenceFileInPathNull);
-
- LongWritable value1 = new LongWritable();
- SequenceFile.Writer writer1 = null;
- try {
- writer1 = SequenceFile.createWriter( fs, conf, path, NullWritable.class, value1.getClass());
- for (int i = 0; i < kvCount; i ++) {
- value1.set(i);
- writer1.append(NullWritable.get(), value1);
- }
- } finally {
- IOUtils.closeStream(writer1);
- }
- }
-
- @Override
- protected void testProgram() throws Exception {
- expectedResult = HadoopIOFormatPrograms.runProgram(curProgId, resultPath, sequenceFileInPath, sequenceFileInPathNull);
- }
-
- @Override
- protected void postSubmit() throws Exception {
- for(int i = 0; i < resultPath.length; i++) {
- compareResultsByLinesInMemory(expectedResult[i], resultPath[i]);
- }
- }
-
- @Parameters
- public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {
-
- LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
-
- for(int i=1; i <= NUM_PROGRAMS; i++) {
- Configuration config = new Configuration();
- config.setInteger("ProgramId", i);
- tConfigs.add(config);
- }
-
- return toParameterList(tConfigs);
- }
-
- public static class HadoopIOFormatPrograms {
-
- public static String[] runProgram(int progId, String resultPath[], String sequenceFileInPath, String sequenceFileInPathNull) throws Exception {
-
- switch(progId) {
- case 1: {
- /**
- * Test sequence file, including a key access.
- */
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- SequenceFileInputFormat<LongWritable, Text> sfif = new SequenceFileInputFormat<LongWritable, Text>();
- JobConf hdconf = new JobConf();
- SequenceFileInputFormat.addInputPath(hdconf, new Path(sequenceFileInPath));
- HadoopInputFormat<LongWritable, Text> hif = new HadoopInputFormat<LongWritable, Text>(sfif, LongWritable.class, Text.class, hdconf);
- DataSet<Tuple2<LongWritable, Text>> ds = env.createInput(hif);
- DataSet<Tuple2<Long, Text>> sumed = ds.map(new MapFunction<Tuple2<LongWritable, Text>, Tuple2<Long, Text>>() {
- @Override
- public Tuple2<Long, Text> map(Tuple2<LongWritable, Text> value) throws Exception {
- return new Tuple2<Long, Text>(value.f0.get(), value.f1);
- }
- }).sum(0);
- sumed.writeAsText(resultPath[0]);
- DataSet<String> res = ds.distinct(0).map(new MapFunction<Tuple2<LongWritable, Text>, String>() {
- @Override
- public String map(Tuple2<LongWritable, Text> value) throws Exception {
- return value.f1 + " - " + value.f0.get();
- }
- });
- res.writeAsText(resultPath[1]);
- env.execute();
-
- // return expected result
- return new String [] {"(21,3 - somestring)", "0 - somestring - 0\n" +
- "1 - somestring - 1\n" +
- "2 - somestring - 2\n" +
- "3 - somestring - 3\n"};
-
- }
- case 2: {
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- SequenceFileInputFormat<NullWritable, LongWritable> sfif = new SequenceFileInputFormat<NullWritable, LongWritable>();
- JobConf hdconf = new JobConf();
- SequenceFileInputFormat.addInputPath(hdconf, new Path(sequenceFileInPathNull));
- HadoopInputFormat<NullWritable, LongWritable> hif = new HadoopInputFormat<NullWritable, LongWritable>(sfif, NullWritable.class, LongWritable.class, hdconf);
- DataSet<Tuple2<NullWritable, LongWritable>> ds = env.createInput(hif);
- DataSet<Tuple2<Void, Long>> res = ds.map(new MapFunction<Tuple2<NullWritable, LongWritable>, Tuple2<Void, Long>>() {
- @Override
- public Tuple2<Void, Long> map(Tuple2<NullWritable, LongWritable> value) throws Exception {
- return new Tuple2<Void, Long>(null, value.f1.get());
- }
- });
- DataSet<Tuple2<Void, Long>> res1 = res.groupBy(1).sum(1);
- res1.writeAsText(resultPath[1]);
- res.writeAsText(resultPath[0]);
- env.execute();
-
- // return expected result
- return new String [] {"(null,2)\n" +
- "(null,0)\n" +
- "(null,1)\n" +
- "(null,3)",
- "(null,0)\n" +
- "(null,1)\n" +
- "(null,2)\n" +
- "(null,3)"};
- }
- default:
- throw new IllegalArgumentException("Invalid program id");
- }
-
- }
-
- }
-}
[04/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/util/TwitterStreamData.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/util/TwitterStreamData.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/util/TwitterStreamData.java
new file mode 100644
index 0000000..0e7976c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/util/TwitterStreamData.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.twitter.util;
+
+public class TwitterStreamData {
+ public static final String[] TEXTS = new String[] {
+ "{\"delete\":{\"status\":{\"id\":448896216110080001,\"id_str\":\"448896216110080001\",\"user_id\":1586900228,\"user_id_str\":\"1586900228\"},\"timestamp_ms\":\"1412178007914\"}}",
+ "{\"created_at\":\"Wed Oct 01 15:40:07 +0000 2014\",\"id\":517338176713523201,\"id_str\":\"517338176713523201\",\"text\":\"RT @IniAnwarHadi: If you're naive enough to think that we would be better off if the opposition won the last elections, then good for you.\",\"source\":\"\\u003ca href=\\\"https:\\/\\/twitter.com\\/download\\/android\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Android\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":1402119228,\"id_str\":\"1402119228\",\"name\":\"\\u6cd5 \\u8482 \\u54c8\",\"screen_name\":\"sesepiSepi\",\"location\":\"Malaysia || Edinburgh\",\"url\":null,\"description\":\"To whom are you enslaving yourself?\",\"protected\":false,\"verified\":false,\"followers_count\":255,\"friends_count\":197,\"listed_count\":2,\"favourites_count\":4064,\"statuses_count\":30382,\"created_at\":\
"Sat May 04 12:01:31 +0000 2013\",\"utc_offset\":28800,\"time_zone\":\"Beijing\",\"geo_enabled\":true,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"FF6699\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/436368673033306112\\/gR_FAlFW.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/436368673033306112\\/gR_FAlFW.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"B40B43\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/510951209868337152\\/6eNXVNb__normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/510951209868337152\\/6eNXVNb__normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banner
s\\/1402119228\\/1409317654\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:17:59 +0000 2014\",\"id\":517332604706508800,\"id_str\":\"517332604706508800\",\"text\":\"If you're naive enough to think that we would be better off if the opposition won the last elections, then good for you.\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":234395084,\"id_str\":\"234395084\",\"name\":\"Anwar Hadi\",\"screen_name\":\"IniAnwarHadi\",\"location\":\"Pulau Pinang\",\"url\":\"http:\\/\\/www.anakazman.b
logspot.com\",\"description\":\"Sometimes a blogger, sometimes a vlogger, sometimes a noob teacher, always a Muslim. Managed by Gushcloud. @DeludeOfficial\",\"protected\":false,\"verified\":false,\"followers_count\":102098,\"friends_count\":289,\"listed_count\":210,\"favourites_count\":186,\"statuses_count\":45071,\"created_at\":\"Wed Jan 05 15:15:07 +0000 2011\",\"utc_offset\":28800,\"time_zone\":\"Kuala Lumpur\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_u
rl\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/456689283240189952\\/ldMM1VUP_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/456689283240189952\\/ldMM1VUP_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/234395084\\/1349837992\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":75,\"favorite_count\":42,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"en\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"IniAnwarHadi\",\"name\":\"Anwar Hadi\",\"id\":234395084,\"id_str\":\"234395084\",\"indices\":[3,16]}],\"s
ymbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_ms\":\"1412178007807\"}",
+ "{\"delete\":{\"status\":{\"id\":516592555278872576,\"id_str\":\"516592555278872576\",\"user_id\":2835411134,\"user_id_str\":\"2835411134\"},\"timestamp_ms\":\"1412178007920\"}}",
+ "{\"created_at\":\"Wed Oct 01 15:40:07 +0000 2014\",\"id\":517338176709345280,\"id_str\":\"517338176709345280\",\"text\":\"RT @takaaaki0410: \\u697d\\u3057\\u305d\\u3046\\u306b\\u9699\\u9593\\u30c1\\u30a7\\u30c3\\u30af\\u3092\\u3059\\u308b\\u3061\\u3073\\u3046\\u305d\\u30ba\\u3002\\u304b\\u308f\\u3046\\u305d\\u3089\\u3057\\u304f\\u306a\\u3063\\u3066\\u304d\\u307e\\u3057\\u305f\\u3002#\\u30b5\\u30f3\\u30b7\\u30e3\\u30a4\\u30f3\\u6c34\\u65cf\\u9928 http:\\/\\/t.co\\/McsvrHPAr0\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\\" rel=\\\"nofollow\\\"\\u003eTwitter Web Client\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":118319345,\"id_str\":\"118319345\",\"name\":\"leonardo\",\"screen_name\":\"masumind\",\"location\":\"Japan\",\"url\":null,\"description\":\"Working for strategic designing. Once lived in U.S
.A., U.S.S.R. and Russia. Now in Japan. Part-time university lecturer \\u500b\\u4eba\\u306e\\u7acb\\u5834\\u3067\\u3064\\u3076\\u3084\\u3044\\u3066\\u3044\\u307e\\u3059\\u3002\",\"protected\":false,\"verified\":false,\"followers_count\":1604,\"friends_count\":597,\"listed_count\":131,\"favourites_count\":600,\"statuses_count\":27957,\"created_at\":\"Sun Feb 28 08:33:20 +0000 2010\",\"utc_offset\":32400,\"time_zone\":\"Tokyo\",\"geo_enaboutputPathled\":false,\"lang\":\"ja\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"ACDED6\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/282110252\\/IMG_4869.JPG\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/282110252\\/IMG_4869.JPG\",\"profile_background_tile\":true,\"profile_link_color\":\"038543\",\"profile_sidebar_border_color\":\"EEEEEE\",\"profile_sidebar_fill_color\":\"F6F6F6\",\"profile_text_color\":\"33333
3\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/1423205970\\/2006_8___________103_normal.jpg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/1423205970\\/2006_8___________103_normal.jpg\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 14:17:39 +0000 2014\",\"id\":517317423549018112,\"id_str\":\"517317423549018112\",\"text\":\"\\u697d\\u3057\\u305d\\u3046\\u306b\\u9699\\u9593\\u30c1\\u30a7\\u30c3\\u30af\\u3092\\u3059\\u308b\\u3061\\u3073\\u3046\\u305d\\u30ba\\u3002\\u304b\\u308f\\u3046\\u305d\\u3089\\u3057\\u304f\\u306a\\u3063\\u3066\\u304d\\u307e\\u3057\\u305f\\u3002#\\u30b5\\u30f3\\u30b7\\u30e3\\u30a4\\u30f3\\u6c34\\u65cf\\u9928 http:\\/\\/t.co\\/McsvrHPAr0\",\"source\":\"\\u003ca href=\\\"ht
tp:\\/\\/twitter.com\\\" rel=\\\"nofollow\\\"\\u003eTwitter Web Client\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":265723969,\"id_str\":\"265723969\",\"name\":\"Takaaki Sawada\",\"screen_name\":\"takaaaki0410\",\"location\":\"\",\"url\":null,\"description\":\"\\u304a\\u3044\\u3057\\u3044\\u3082\\u306e\\u3068\\u30ab\\u30ef\\u30a6\\u30bd\\u304c\\u597d\\u304d\\u3067\\u3059\\u3002\",\"protected\":false,\"verified\":false,\"followers_count\":352,\"friends_count\":197,\"listed_count\":13,\"favourites_count\":366,\"statuses_count\":6634,\"created_at\":\"Mon Mar 14 01:54:33 +0000 2011\",\"utc_offset\":32400,\"time_zone\":\"Tokyo\",\"geo_enabled\":false,\"lang\":\"ja\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"000000\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com
\\/profile_background_images\\/749259666\\/9ebe30d2ac8d95d6266181ab0f61f90c.png\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/749259666\\/9ebe30d2ac8d95d6266181ab0f61f90c.png\",\"profile_background_tile\":true,\"profile_link_color\":\"599BFF\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"585959\",\"profile_text_color\":\"FF82E8\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/415878656800460801\\/OSLSTJcC_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/415878656800460801\\/OSLSTJcC_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/265723969\\/1398242024\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_coun
t\":69,\"favorite_count\":79,\"entities\":{\"hashtags\":[{\"text\":\"\\u30b5\\u30f3\\u30b7\\u30e3\\u30a4\\u30f3\\u6c34\\u65cf\\u9928\",\"indices\":[35,45]}],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[],\"media\":[{\"id\":517317416271880192,\"id_str\":\"517317416271880192\",\"indices\":[46,68],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}}}]},\"extended_entities\":{\"media\":[{\"id\":517317416271880192,\"id_str\":\"517317416271880192\",\"indi
ces\":[46,68],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}}},{\"id\":517317413965004800,\"id_str\":\"517317413965004800\",\"indices\":[46,68],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcneCAAA-HPL.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcneCAAA-HPL.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/51731742
3549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}}},{\"id\":517317407241543681,\"id_str\":\"517317407241543681\",\"indices\":[46,68],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcObCIAEm57j.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcObCIAEm57j.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}}}]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":fals
e,\"filter_level\":\"low\",\"lang\":\"ja\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"\\u30b5\\u30f3\\u30b7\\u30e3\\u30a4\\u30f3\\u6c34\\u65cf\\u9928\",\"indices\":[53,63]}],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"takaaaki0410\",\"name\":\"Takaaki Sawada\",\"id\":265723969,\"id_str\":\"265723969\",\"indices\":[3,16]}],\"symbols\":[],\"media\":[{\"id\":517317416271880192,\"id_str\":\"517317416271880192\",\"indices\":[64,86],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":15
0,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}},\"source_status_id\":517317423549018112,\"source_status_id_str\":\"517317423549018112\"}]},\"extended_entities\":{\"media\":[{\"id\":517317416271880192,\"id_str\":\"517317416271880192\",\"indices\":[64,86],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}},\"source_status_id\":517317423549018112,\"source_status_id_str\":\"517317423549018112\"},{\"id\":517317413965004800,\"id_str\":\"51731
7413965004800\",\"indices\":[64,86],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcneCAAA-HPL.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcneCAAA-HPL.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}},\"source_status_id\":517317423549018112,\"source_status_id_str\":\"517317423549018112\"},{\"id\":517317407241543681,\"id_str\":\"517317407241543681\",\"indices\":[64,86],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcObCIAEm57j.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcObCIAEm57j.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\"
:\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}},\"source_status_id\":517317423549018112,\"source_status_id_str\":\"517317423549018112\"}]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"ja\",\"timestamp_ms\":\"1412178007863\"}",
+ "{\"delete\":{\"status\":{\"id\":517192344945192960,\"id_str\":\"517192344945192960\",\"user_id\":1134175530,\"user_id_str\":\"1134175530\"},\"timestamp_ms\":\"1412178007972\"}}",
+ "{\"delete\":{\"status\":{\"id\":431653696582078464,\"id_str\":\"431653696582078464\",\"user_id\":1028327856,\"user_id_str\":\"1028327856\"},\"timestamp_ms\":\"1412178007990\"}}",
+ "{\"delete\":{\"status\":{\"id\":517337912464408576,\"id_str\":\"517337912464408576\",\"user_id\":607266683,\"user_id_str\":\"607266683\"},\"timestamp_ms\":\"1412178008106\"}}",
+ "{\"delete\":{\"status\":{\"id\":517191707427758080,\"id_str\":\"517191707427758080\",\"user_id\":268787345,\"user_id_str\":\"268787345\"},\"timestamp_ms\":\"1412178008127\"}}",
+ "{\"delete\":{\"status\":{\"id\":382158053065256961,\"id_str\":\"382158053065256961\",\"user_id\":1303521211,\"user_id_str\":\"1303521211\"},\"timestamp_ms\":\"1412178008294\"}}",
+ "{\"delete\":{\"status\":{\"id\":421056787530280960,\"id_str\":\"421056787530280960\",\"user_id\":63286510,\"user_id_str\":\"63286510\"},\"timestamp_ms\":\"1412178008377\"}}",
+ "{\"delete\":{\"status\":{\"id\":450659899941584898,\"id_str\":\"450659899941584898\",\"user_id\":257093107,\"user_id_str\":\"257093107\"},\"timestamp_ms\":\"1412178008454\"}}",
+ "{\"delete\":{\"status\":{\"id\":450515712348790784,\"id_str\":\"450515712348790784\",\"user_id\":257093107,\"user_id_str\":\"257093107\"},\"timestamp_ms\":\"1412178008458\"}}",
+ "{\"delete\":{\"status\":{\"id\":517190671422087168,\"id_str\":\"517190671422087168\",\"user_id\":1147896566,\"user_id_str\":\"1147896566\"},\"timestamp_ms\":\"1412178008435\"}}",
+ "{\"delete\":{\"status\":{\"id\":324613502436257793,\"id_str\":\"324613502436257793\",\"user_id\":634892640,\"user_id_str\":\"634892640\"},\"timestamp_ms\":\"1412178008499\"}}",
+ "{\"delete\":{\"status\":{\"id\":497003314308186112,\"id_str\":\"497003314308186112\",\"user_id\":451705771,\"user_id_str\":\"451705771\"},\"timestamp_ms\":\"1412178008523\"}}",
+ "{\"delete\":{\"status\":{\"id\":516590386823696385,\"id_str\":\"516590386823696385\",\"user_id\":1556958698,\"user_id_str\":\"1556958698\"},\"timestamp_ms\":\"1412178008623\"}}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180895662080,\"id_str\":\"517338180895662080\",\"text\":\"yolunu bulmaktan umudunu kesme, \\u00e7\\u00fcnk\\u00fc umut,tur insana yol g\\u00f6steren\\n G\\u00f6klerdeY\\u0131ld\\u0131zKalplerdeAy T\\u00fcrkiyedirGalatasaray\",\"source\":\"\\u003ca href=\\\"https:\\/\\/twitter.com\\/\\\" rel=\\\"nofollow\\\"\\u003eMobile Webs\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":712720935,\"id_str\":\"712720935\",\"name\":\"Bensuu\",\"screen_name\":\"bnsukara\",\"location\":\"YASEM\\u0130N HER\\u015eEY\\u0130M\",\"url\":\"https:\\/\\/www.facebook.com\\/Webmaster99\",\"description\":\"EMRE POLAT TAK\\u0130P EDEN TAK\\u0130PC\\u0130 KAZANIYOR ----@emrepolatt1\",\"protected\":false,\"verified\":false,\"followers_count\":422,\"friends_count\":327,\"listed_coun
t\":0,\"favourites_count\":4960,\"statuses_count\":3561,\"created_at\":\"Sun Oct 13 14:14:03 +0000 2013\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"tr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/378800000113509334\\/492598cb8f47b2a29fe392d7e9eabb44.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/378800000113509334\\/492598cb8f47b2a29fe392d7e9eabb44.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/516711945240838144\\/1MXxVPoJ_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\
\/516711945240838144\\/1MXxVPoJ_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/712720935\\/1402706181\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"tr\",\"timestamp_ms\":\"1412178008664\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180887261184,\"id_str\":\"517338180887261184\",\"text\":\"#Bug\\u00fcnG\\u00fcnlerdenAvrupan\\u0131nFethiGALATASARAY\\u0131m\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":295319165,\"id_str\":\"295319165\",\"name\":\"Do\\u011fa Marangoz 1905\",\"screen_name\":\"doa1905\",\"location\":\"\",\"url\":null,\"description\":\"Cehennem donana kadar #GALATASARAY N.K.F.V.A.S -Sar\\u0131yer Do\\u011fa Koleji #HamitAltintop #FelipeMelo #BurakY\\u0131lmaz #OlcanAd\\u0131n 30.03.2012 T.D OMERNAZ \\u0130stanbul\\/Sar\\u0131yer\",\"protected\":false,\"verified\":false,\"followers_count\":578,\"friends_count\":272,\"listed_count\":0,\"favou
rites_count\":7928,\"statuses_count\":12005,\"created_at\":\"Sun May 08 19:51:11 +0000 2011\",\"utc_offset\":-18000,\"time_zone\":\"Quito\",\"geo_enabled\":false,\"lang\":\"tr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/814478464\\/5ea45052ee5cdc7d424ede463bd88795.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/814478464\\/5ea45052ee5cdc7d424ede463bd88795.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/513713202191163392\\/dKiNGqS-_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/513713202191163392\\/
dKiNGqS-_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/295319165\\/1397810158\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"Bug\\u00fcnG\\u00fcnlerdenAvrupan\\u0131nFethiGALATASARAY\\u0131m\",\"indices\":[0,42]}],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"und\",\"timestamp_ms\":\"1412178008666\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180883058688,\"id_str\":\"517338180883058688\",\"text\":\"Un par de zapatos, dos estilos http:\\/\\/t.co\\/SOGGZDmTrk\",\"source\":\"\\u003ca href=\\\"http:\\/\\/www.wonderland.fm\\/\\\" rel=\\\"nofollow\\\"\\u003ewonderland.fm App\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":1135415474,\"id_str\":\"1135415474\",\"name\":\"CalzandoZapatos\",\"screen_name\":\"CalzandoZapatos\",\"location\":\"\",\"url\":null,\"description\":\"Sobre todo tipo de zapatos y calzado. Encontraras lo m\\u00e1s nuevo en noticias, y todo tipo de ofertas\",\"protected\":false,\"verified\":false,\"followers_count\":10181,\"friends_count\":10975,\"listed_count\":15,\"favourites_count\":0,\"statuses_count\":5434,\"created_at\":\"Wed Jan 30 22:51:24 +0000 2013\",\"utc_offset\":
7200,\"time_zone\":\"Amsterdam\",\"geo_enabled\":false,\"lang\":\"es\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"D1B6D1\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme17\\/bg.gif\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme17\\/bg.gif\",\"profile_background_tile\":false,\"profile_link_color\":\"CC3366\",\"profile_sidebar_border_color\":\"000000\",\"profile_sidebar_fill_color\":\"E6F6F9\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":false,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/3197686515\\/08837c6a23605352e2bd73c7890cb54d_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/3197686515\\/08837c6a23605352e2bd73c7890cb54d_normal.jpeg\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\
":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[{\"url\":\"http:\\/\\/t.co\\/SOGGZDmTrk\",\"expanded_url\":\"http:\\/\\/bit.ly\\/1rGjI18\",\"display_url\":\"bit.ly\\/1rGjI18\",\"indices\":[31,53]}],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"es\",\"timestamp_ms\":\"1412178008663\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180887257088,\"id_str\":\"517338180887257088\",\"text\":\"\\u0412\\u043c\\u0435\\u0441\\u0442\\u043e \\u044d\\u0442\\u043e\\u0433\\u043e \\u0440\\u043e\\u0434\\u0438\\u0442\\u0435\\u043b\\u0438 \\u0434\\u043e\\u043b\\u0436\\u043d\\u044b \\u043f\\u0435\\u0440\\u0435\\u043d\\u0430\\u043f\\u0440\\u0430\\u0432\\u0438\\u0442\\u044c \\u0434\\u0435\\u0442\\u0435\\u0439.\",\"source\":\"\\u003ca href=\\\"http:\\/\\/barlear.ru\\/\\\" rel=\\\"nofollow\\\"\\u003eSocialNetworkMarketingApp\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":350347678,\"id_str\":\"350347678\",\"name\":\"murat g\\u00fcndo\\u011fdu\",\"screen_name\":\"mrt_mrt_\",\"location\":\"\",\"url\":null,\"description\":null,\"protected\":false,\"verified\":false,\"followers_count\":369,\"friends_c
ount\":387,\"listed_count\":1,\"favourites_count\":0,\"statuses_count\":1691,\"created_at\":\"Sun Aug 07 16:36:54 +0000 2011\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"tr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/1482992087\\/58537_148894648483546_100000890344526_246985_5158298_n_normal.jpg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/1482992087\\/58537_148894648483546_1000008903
44526_246985_5158298_n_normal.jpg\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"ru\",\"timestamp_ms\":\"1412178008664\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180878860288,\"id_str\":\"517338180878860288\",\"text\":\"RT @jennybethm: Illegal Immigration: 70 Percent of Released Families Never Report Back To Federal Agents http:\\/\\/t.co\\/89DNq2IUFK #teaparty\",\"source\":\"\\u003ca href=\\\"https:\\/\\/roundteam.co\\\" rel=\\\"nofollow\\\"\\u003eRoundTeam\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":596541160,\"id_str\":\"596541160\",\"name\":\"Choose Freedom\",\"screen_name\":\"choose__freedom\",\"location\":\"Where they can't find me.\",\"url\":null,\"description\":\"I believe in #freedom, limited government, a strong national defense, secure borders + personal responsibility. #tcot #tlot #PJNET #NRA #2A #TeaParty #TGDN\",\"protected\":false,\"verified\":false,\"followers_count\":1552,\"friends_count
\":1237,\"listed_count\":34,\"favourites_count\":30,\"statuses_count\":44052,\"created_at\":\"Fri Jun 01 15:05:49 +0000 2012\",\"utc_offset\":-14400,\"time_zone\":\"Eastern Time (US & Canada)\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/445184233460752385\\/RTBwpDuJ_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/445184233460752385\\/RTBwpDuJ_normal.jpeg\",\"profile_
banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/596541160\\/1408567298\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:21:13 +0000 2014\",\"id\":517333419173609472,\"id_str\":\"517333419173609472\",\"text\":\"Illegal Immigration: 70 Percent of Released Families Never Report Back To Federal Agents http:\\/\\/t.co\\/89DNq2IUFK #teaparty\",\"source\":\"\\u003ca href=\\\"http:\\/\\/bufferapp.com\\\" rel=\\\"nofollow\\\"\\u003eBuffer\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":6742412,\"id_str\":\"6742412\",\"name\":\"Jenny Beth Martin\",\"screen_name\":\"jennybethm\",\"location\":\"Atlanta, Ga\",\"ur
l\":\"http:\\/\\/jenuinejen.com\",\"description\":\"Co-Founder Tea Party Patriots; Tips, Tales, and Thoughts of Peach State Mom of Twins; Christian; Conservative; was Republican now American #tcot #teaparty #tgdn\",\"protected\":false,\"verified\":true,\"followers_count\":42320,\"friends_count\":23511,\"listed_count\":717,\"favourites_count\":570,\"statuses_count\":6266,\"created_at\":\"Mon Jun 11 14:32:10 +0000 2007\",\"utc_offset\":-14400,\"time_zone\":\"Eastern Time (US & Canada)\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"000000\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/95371301\\/new_fake_large_175-4.jpg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/95371301\\/new_fake_large_175-4.jpg\",\"profile_background_tile\":false,\"profile_link_color\":\"D60000\",\"profile_sidebar_border_color\":\"000000\"
,\"profile_sidebar_fill_color\":\"1C1939\",\"profile_text_color\":\"777777\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/2281265352\\/7e3d1pifsn0tjy895f5q_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/2281265352\\/7e3d1pifsn0tjy895f5q_normal.jpeg\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":4,\"favorite_count\":3,\"entities\":{\"hashtags\":[{\"text\":\"teaparty\",\"indices\":[112,121]}],\"trends\":[],\"urls\":[{\"url\":\"http:\\/\\/t.co\\/89DNq2IUFK\",\"expanded_url\":\"http:\\/\\/buff.ly\\/1nGdCzB\",\"display_url\":\"buff.ly\\/1nGdCzB\",\"indices\":[89,111]}],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"en\"},\
"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"teaparty\",\"indices\":[128,137]}],\"trends\":[],\"urls\":[{\"url\":\"http:\\/\\/t.co\\/89DNq2IUFK\",\"expanded_url\":\"http:\\/\\/buff.ly\\/1nGdCzB\",\"display_url\":\"buff.ly\\/1nGdCzB\",\"indices\":[105,127]}],\"user_mentions\":[{\"screen_name\":\"jennybethm\",\"name\":\"Jenny Beth Martin\",\"id\":6742412,\"id_str\":\"6742412\",\"indices\":[3,14]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_ms\":\"1412178008676\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180899864576,\"id_str\":\"517338180899864576\",\"text\":\"RT @ExtinctLana90: Je suis 100% d'accord pour mettre #YEnAVraimentMarreDesTypesQuiFontDesHashtag\\u00e0RallongeHeureusementQueTuMetsDesMajuscule\\u2026\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2245575537,\"id_str\":\"2245575537\",\"name\":\"krikristoophe\",\"screen_name\":\"krikristoophe\",\"location\":\"quelque part\",\"url\":null,\"description\":\"snapchat: christophe22470\\r\\ninstagram : @christophe22470 (trop d'imagination)\",\"protected\":false,\"verified\":false,\"followers_count\":393,\"friends_count\":370,\"listed_count\":1,\"favourites_count\":402,\"s
tatuses_count\":8458,\"created_at\":\"Thu Dec 26 23:27:14 +0000 2013\",\"utc_offset\":7200,\"time_zone\":\"Paris\",\"geo_enabled\":true,\"lang\":\"fr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"31A81C\",\"profile_sidebar_border_color\":\"000000\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/510763955930021888\\/RWzT-EhH_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/510763955930021888\\/RWzT-EhH_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/2245575537\\/140770
2870\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:31:10 +0000 2014\",\"id\":517335924968292353,\"id_str\":\"517335924968292353\",\"text\":\"Je suis 100% d'accord pour mettre #YEnAVraimentMarreDesTypesQuiFontDesHashtag\\u00e0RallongeHeureusementQueTuMetsDesMajusculesHein en TT :D\",\"source\":\"\\u003ca href=\\\"http:\\/\\/www.meltingmots.com\\\" rel=\\\"nofollow\\\"\\u003eExtinctLana90\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2593022185,\"id_str\":\"2593022185\",\"name\":\"Lana\",\"screen_name\":\"ExtinctLana90\",\"location\":\"\",\"url\":null,\"description\":\"Lana, pour vous servir. Ou pas.\\r\\n#T
eamInsomniaque.\",\"protected\":false,\"verified\":false,\"followers_count\":487,\"friends_count\":893,\"listed_count\":0,\"favourites_count\":1,\"statuses_count\":1829,\"created_at\":\"Sat Jun 28 12:14:50 +0000 2014\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"fr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/482916277862662145\\/iylCxj4N_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile
_images\\/482916277862662145\\/iylCxj4N_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/2593022185\\/1403971199\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":1,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"YEnAVraimentMarreDesTypesQuiFontDesHashtag\\u00e0RallongeHeureusementQueTuMetsDesMajusculesHein\",\"indices\":[35,125]}],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"fr\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"YEnAVraimentMarreDesTypesQuiFontDesHashtag\\u00e0RallongeHeureusementQueTuMetsDesMajusculesHein\",\"indices\":[54,140]}],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"Exti
nctLana90\",\"name\":\"Lana\",\"id\":2593022185,\"id_str\":\"2593022185\",\"indices\":[3,17]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"fr\",\"timestamp_ms\":\"1412178008683\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180912427008,\"id_str\":\"517338180912427008\",\"text\":\"RT @r1_follow1: \\u0628\\u064a\\u0639 \\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646 \\u062e\\u0644\\u064a\\u062c\\u064a\\u0646 \\u0644\\u0644\\u062a\\u0648\\u064a\\u062a\\u0631 \\u0648 \\u0625\\u0646\\u0633\\u062a\\u0642\\u0631\\u0627\\u0645 \\n\\n\\u0644\\u0644\\u062a\\u0648\\u0627\\u0635\\u0644 \\u0648\\u0627\\u062a\\u0633 \\u0627\\u0628 0596168454\\n\\nhttp:\\/\\/t.co\\/JzuAqNDCbV \\n 749\",\"source\":\"\\u003ca href=\\\"http:\\/\\/ffasr3.com\\/2\\\" rel=\\\"nofollow\\\"\\u003e\\u0627\\u0644\\u062a\\u0648\\u064a\\u062a\\u0631\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2650616521,\"id_str\":\"2650616521\",\"name\":\"quteba usma alani\",\"screen_name\":\"QutebaUsma\",\"location\":\"
\",\"url\":null,\"description\":null,\"protected\":false,\"verified\":false,\"followers_count\":9,\"friends_count\":40,\"listed_count\":0,\"favourites_count\":6,\"statuses_count\":2086,\"created_at\":\"Wed Jul 16 09:51:04 +0000 2014\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/abs.twimg.com\\/sticky\\/default_profile_images\\/default_profile_0_normal.png\",\"profile_image_url_https\":\"https:\\/\\/abs
.twimg.com\\/sticky\\/default_profile_images\\/default_profile_0_normal.png\",\"default_profile\":true,\"default_profile_image\":true,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:36:11 +0000 2014\",\"id\":517337187596390400,\"id_str\":\"517337187596390400\",\"text\":\"\\u0628\\u064a\\u0639 \\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646 \\u062e\\u0644\\u064a\\u062c\\u064a\\u0646 \\u0644\\u0644\\u062a\\u0648\\u064a\\u062a\\u0631 \\u0648 \\u0625\\u0646\\u0633\\u062a\\u0642\\u0631\\u0627\\u0645 \\n\\n\\u0644\\u0644\\u062a\\u0648\\u0627\\u0635\\u0644 \\u0648\\u0627\\u062a\\u0633 \\u0627\\u0628 0596168454\\n\\nhttp:\\/\\/t.co\\/JzuAqNDCbV \\n 749\",\"source\":\"\\u003ca href=\\\"http:\\/\\/ffasr3.com\\/a\\\" rel=\\\"nofollow\\\"\\u003etweet \\/a\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_i
d_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2800103305,\"id_str\":\"2800103305\",\"name\":\"\\u0628\\u064a\\u0639 \\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646\",\"screen_name\":\"r1_follow1\",\"location\":\"\",\"url\":null,\"description\":\"\\u200f\\u200f\\u0628\\u064a\\u0639 \\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646 \\u062d\\u0642\\u064a\\u0642\\u064a\\u0646 \\u062e\\u0644\\u064a\\u062c\\u064a\\u0646 \\u0644\\u0644\\u062a\\u0648\\u064a\\u062a\\u0631 \\u0648 \\u0627\\u0646\\u0633\\u062a\\u063a\\u0631\\u0627\\u0645 \\u0648 \\u0644\\u0627\\u064a\\u0643\\u0627\\u062a \\u062e\\u0644\\u064a\\u062c\\u064a\\u0646 \\u062b\\u0627\\u0628\\u062a\\u064a\\u0646 \\u0645\\u0639 \\u0636\\u0645\\u0627\\u0646 \\u0644\\u0643 \\u0627\\u0644\\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646 \\u0648\\u0627\\u062a\\u0633 \\u0627\\u0628 0596168454\",\"protected\":false,\"verified\":false,\"followers_count\":7704,\"
friends_count\":5500,\"listed_count\":4,\"favourites_count\":3,\"statuses_count\":3089,\"created_at\":\"Tue Sep 09 15:02:39 +0000 2014\",\"utc_offset\":3600,\"time_zone\":\"London\",\"geo_enabled\":false,\"lang\":\"fr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/509428053143846912\\/gAG3VoKW_normal.png\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/509428053143846912\\/gAG3VoKW_normal.png\",\"profile_banner_url\":\
"https:\\/\\/pbs.twimg.com\\/profile_banners\\/2800103305\\/1410292101\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":1517,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[],\"media\":[{\"id\":515942230847021056,\"id_str\":\"515942230847021056\",\"indices\":[68,90],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"url\":\"http:\\/\\/t.co\\/JzuAqNDCbV\",\"display_url\":\"pic.twitter.com\\/JzuAqNDCbV\",\"expanded_url\":\"http:\\/\\/twitter.com\\/ilyass_h\\/status\\/515942234248998912\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"medium\":{\"w\":598,\
"h\":352,\"resize\":\"fit\"},\"small\":{\"w\":340,\"h\":200,\"resize\":\"fit\"}},\"source_status_id\":515942234248998912,\"source_status_id_str\":\"515942234248998912\"}]},\"extended_entities\":{\"media\":[{\"id\":515942230847021056,\"id_str\":\"515942230847021056\",\"indices\":[68,90],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"url\":\"http:\\/\\/t.co\\/JzuAqNDCbV\",\"display_url\":\"pic.twitter.com\\/JzuAqNDCbV\",\"expanded_url\":\"http:\\/\\/twitter.com\\/ilyass_h\\/status\\/515942234248998912\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"medium\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"small\":{\"w\":340,\"h\":200,\"resize\":\"fit\"}},\"source_status_id\":515942234248998912,\"source_status_id_str\":\"515942234248998912\"}]},\"favorited\":false,\"retweeted\":false,\"
possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"ar\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"r1_follow1\",\"name\":\"\\u0628\\u064a\\u0639 \\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646\",\"id\":2800103305,\"id_str\":\"2800103305\",\"indices\":[3,14]}],\"symbols\":[],\"media\":[{\"id\":515942230847021056,\"id_str\":\"515942230847021056\",\"indices\":[84,106],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"url\":\"http:\\/\\/t.co\\/JzuAqNDCbV\",\"display_url\":\"pic.twitter.com\\/JzuAqNDCbV\",\"expanded_url\":\"http:\\/\\/twitter.com\\/ilyass_h\\/status\\/515942234248998912\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"medium\":{\"w\":598,\"h\":352,\"resize\":\"fit\
"},\"small\":{\"w\":340,\"h\":200,\"resize\":\"fit\"}},\"source_status_id\":515942234248998912,\"source_status_id_str\":\"515942234248998912\"}]},\"extended_entities\":{\"media\":[{\"id\":515942230847021056,\"id_str\":\"515942230847021056\",\"indices\":[84,106],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"url\":\"http:\\/\\/t.co\\/JzuAqNDCbV\",\"display_url\":\"pic.twitter.com\\/JzuAqNDCbV\",\"expanded_url\":\"http:\\/\\/twitter.com\\/ilyass_h\\/status\\/515942234248998912\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"medium\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"small\":{\"w\":340,\"h\":200,\"resize\":\"fit\"}},\"source_status_id\":515942234248998912,\"source_status_id_str\":\"515942234248998912\"}]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":fals
e,\"filter_level\":\"medium\",\"lang\":\"ar\",\"timestamp_ms\":\"1412178008686\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180891467776,\"id_str\":\"517338180891467776\",\"text\":\"RT @K31L4_: @oliviazecchin waaaaaaa que decis personaje jajajaja\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/android\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Android\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":1473191491,\"id_str\":\"1473191491\",\"name\":\"Oli\",\"screen_name\":\"oliviazecchin\",\"location\":\"\",\"url\":null,\"description\":\"Y la verdad, no hay cosa peor, \\nque vivir sin pasi\\u00f3n..\",\"protected\":false,\"verified\":false,\"followers_count\":360,\"friends_count\":325,\"listed_count\":0,\"favourites_count\":1688,\"statuses_count\":11441,\"created_at\":\"Fri May 31 21:55:06 +0000 2013\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\
":true,\"lang\":\"es\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"ACDED6\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme18\\/bg.gif\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme18\\/bg.gif\",\"profile_background_tile\":false,\"profile_link_color\":\"038543\",\"profile_sidebar_border_color\":\"EEEEEE\",\"profile_sidebar_fill_color\":\"F6F6F6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/509118978350804992\\/Topjl7PV_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/509118978350804992\\/Topjl7PV_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/1473191491\\/1410377782\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notification
s\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:27:42 +0000 2014\",\"id\":517335050082205696,\"id_str\":\"517335050082205696\",\"text\":\"@oliviazecchin waaaaaaa que decis personaje jajajaja\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\\" rel=\\\"nofollow\\\"\\u003eTwitter Web Client\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":517333590489964545,\"in_reply_to_status_id_str\":\"517333590489964545\",\"in_reply_to_user_id\":1473191491,\"in_reply_to_user_id_str\":\"1473191491\",\"in_reply_to_screen_name\":\"oliviazecchin\",\"user\":{\"id\":917988457,\"id_str\":\"917988457\",\"name\":\"cronopio redondito\",\"screen_name\":\"K31L4_\",\"location\":\"Carlos Alberto Garc\\u00eda Moreno\",\"url\":null,\"description\":\"\\u2119\\u211d\\u2655 y sus Redonditos de Ricota y AP en algunos ciclos solares. Club Atl\\u00e9tico Rosario Central. Mi vieja cri\\u00f3 un idiota de coraz\\
u00f3n lun\\u00e1tico.\",\"protected\":false,\"verified\":false,\"followers_count\":97,\"friends_count\":119,\"listed_count\":0,\"favourites_count\":149,\"statuses_count\":4060,\"created_at\":\"Thu Nov 01 01:01:39 +0000 2012\",\"utc_offset\":-10800,\"time_zone\":\"Santiago\",\"geo_enabled\":false,\"lang\":\"es\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"131516\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/469629734930771968\\/-8Z_79Q1.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/469629734930771968\\/-8Z_79Q1.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"42171A\",\"profile_sidebar_border_color\":\"000000\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/478290344656650240\\/cYsZS_tY_norm
al.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/478290344656650240\\/cYsZS_tY_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/917988457\\/1408206021\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":1,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"oliviazecchin\",\"name\":\"Oli\",\"id\":1473191491,\"id_str\":\"1473191491\",\"indices\":[0,14]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"es\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"K31L4_\",\"name\":\"cronopio redondito\",\"id\":917988457,\"id_str\":\"91798
8457\",\"indices\":[3,10]},{\"screen_name\":\"oliviazecchin\",\"name\":\"Oli\",\"id\":1473191491,\"id_str\":\"1473191491\",\"indices\":[12,26]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"es\",\"timestamp_ms\":\"1412178008684\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180912443392,\"id_str\":\"517338180912443392\",\"text\":\"@BD_Lay I Knew It \\ud83d\\ude02\\ud83d\\ude2d\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/android\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Android\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":517335209826848769,\"in_reply_to_status_id_str\":\"517335209826848769\",\"in_reply_to_user_id\":311668757,\"in_reply_to_user_id_str\":\"311668757\",\"in_reply_to_screen_name\":\"BD_Lay\",\"user\":{\"id\":568404382,\"id_str\":\"568404382\",\"name\":\"NUT WORLD BITCH\\u2757\",\"screen_name\":\"Ballout_4250\",\"location\":\"Welcome To Nut World\\u2757 4\\u20e32\\u20e35\\u20e30\\u20e3\",\"url\":null,\"description\":\"Big Ballout FRM The A\\u2757 Free All The Guys \\u2757R.I.P. Hoddy & Nut\\u2757.. Love Viniya Life \\u2757\\ufe0f\\u2764\\ufe0f\",\"protected\":false,\"verified\":false,\"followers_count\":963,\"friends_count\":90
7,\"listed_count\":2,\"favourites_count\":9224,\"statuses_count\":56249,\"created_at\":\"Tue May 01 17:52:53 +0000 2012\",\"utc_offset\":-10800,\"time_zone\":\"Atlantic Time (Canada)\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/378800000039624360\\/ed7ce9d140503e73711e6d71fe6fce1c.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/378800000039624360\\/ed7ce9d140503e73711e6d71fe6fce1c.jpeg\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"000000\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/508187491305009152\\/uE0eB6zG_normal.jpeg\",\"profile_image_url_https\":\"ht
tps:\\/\\/pbs.twimg.com\\/profile_images\\/508187491305009152\\/uE0eB6zG_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/568404382\\/1409189327\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"BD_Lay\",\"name\":\"sheemagang \",\"id\":311668757,\"id_str\":\"311668757\",\"indices\":[0,7]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_ms\":\"1412178008671\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180886872064,\"id_str\":\"517338180886872064\",\"text\":\"\\u5a18\\u3002\\u30b3\\u30f3\\u306e\\u65e5\\u3060\\u3063\\u305f\\u306e\\u306b\\u4f55\\u6545\\u304b\\u8133\\u5185\\u3067\\u306f\\u30d9\\u30ea\\u304c\\u6d41\\u308c\\u3066\\u304a\\u308b\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":132108937,\"id_str\":\"132108937\",\"name\":\"\\u72c2\\u30cf\\u30e0\\u3061\\u3072\\u308d'14\",\"screen_name\":\"chii_hs\",\"location\":\"\\u3055\\u3093\\u308a\\u304a\\u3074\\u3085\\u30fc\\u308d\\u3089\\u3093\\u3069\",\"url\":null,\"description\":\"\\u6b4c\\u3046\\u3053\\u3068\\u3068\\u4f5c\\u308b\\u3053\\u3068\\u304c\\u597d\\u304d\\u3067\\u30
59\\u3002\\u30e9\\u30fc\\u30e1\\u30f3*\\u30b5\\u30f3\\u30ea\\u30aa*\\u30b7\\u30ca\\u30e2\\u30ed\\u30fc\\u30eb*\\u8584\\u685c\\u9b3c*\\u304a\\u6d0b\\u670d*\\u30cf\\u30ed\\u30d7\\u30ed*\\u30e2\\u30fc\\u30cb\\u30f3\\u30b0\\u5a18\\u3002*\",\"protected\":false,\"verified\":false,\"followers_count\":388,\"friends_count\":391,\"listed_count\":15,\"favourites_count\":2259,\"statuses_count\":40945,\"created_at\":\"Mon Apr 12 09:04:44 +0000 2010\",\"utc_offset\":32400,\"time_zone\":\"Tokyo\",\"geo_enabled\":true,\"lang\":\"ja\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"1A1B1F\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme9\\/bg.gif\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme9\\/bg.gif\",\"profile_background_tile\":false,\"profile_link_color\":\"2FC2EF\",\"profile_sidebar_border_color\":\"181A1E\",\"profile_sidebar_fill_color\":\"252429\",\"profile_text_color\
":\"666666\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/458234331832410112\\/UW0e7cCy_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/458234331832410112\\/UW0e7cCy_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/132108937\\/1395030721\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"ja\",\"timestamp_ms\":\"1412178008664\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180904030210,\"id_str\":\"517338180904030210\",\"text\":\"RT @V8888S: \\u0645\\u0633\\u0627\\u0621 \\u0627\\u0644\\u0643\\u0644\\u0645\\u0629 \\u0627\\u0644\\u062d\\u0644\\u0648\\u0629 \\u0645\\u0633\\u0627\\u0621 \\u0627\\u0644\\u062d\\u0628 \\u0628\\u0644\\u064a\\u0627 \\u062d\\u062f\\u0648\\u062f \\u0645\\u0633\\u0627\\u0621 \\u064a\\u062d\\u062a\\u0648\\u064a\\u0647 \\u0627\\u0644\\u0643\\u0648\\u0646 \\u0644\\u0623\\u0646\\u0643 \\u0623\\u0646\\u062a \\u0641\\u064a\\u0647 \\u0645\\u0648\\u062c\\u0648\\u062f \\u0645\\u0633\\u0627\\u0626\\u064a \\u0627\\u0646\\u062a\\u0650 \\u2764\\ufe0f\\ud83d\\udc8b\\ud83c\\udf39\",\"source\":\"\\u003ca href=\\\"http:\\/\\/vsbelle.net\\/120\\/\\\" rel=\\\"nofollow\\\"\\u003ehttp:\\/\\/vsbelle.net\\/120\\/\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_r
eply_to_screen_name\":null,\"user\":{\"id\":2254043439,\"id_str\":\"2254043439\",\"name\":\"Camilla\\r Darcey\\r\",\"screen_name\":\"a_lndem00\",\"location\":\"Texas\\r\",\"url\":null,\"description\":\"\\u0644\\u0644\\u0635\\u062f\\u0627\\u0642\\u0629 (\\u0627\\u0644\\u062e\\u0627\\u0635)\\n\\n\\u0627\\u0644\\u0631\\u064a\\u0627\\u0636\",\"protected\":false,\"verified\":false,\"followers_count\":81,\"friends_count\":521,\"listed_count\":0,\"favourites_count\":2339,\"statuses_count\":24523,\"created_at\":\"Tue Dec 31 11:32:31 +0000 2013\",\"utc_offset\":-25200,\"time_zone\":\"Pacific Time (US & Canada)\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"141414\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme7\\/bg.gif\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme7\\/bg.gif\",\"profile_background_tile\":true,\"profile_link_co
lor\":\"942C2C\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"F3F3F3\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/430389252866592768\\/SWbXjVTn_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/430389252866592768\\/SWbXjVTn_normal.jpeg\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:32:09 +0000 2014\",\"id\":517336173060968448,\"id_str\":\"517336173060968448\",\"text\":\"\\u0645\\u0633\\u0627\\u0621 \\u0627\\u0644\\u0643\\u0644\\u0645\\u0629 \\u0627\\u0644\\u062d\\u0644\\u0648\\u0629 \\u0645\\u0633\\u0627\\u0621 \\u0627\\u0644\\u062d\\u0628 \\u0628\\u0644\\u064a\\u0627 \\u062d\\u062f\\u0648\\u062f \\u0645\\u0633\\
u0627\\u0621 \\u064a\\u062d\\u062a\\u0648\\u064a\\u0647 \\u0627\\u0644\\u0643\\u0648\\u0646 \\u0644\\u0623\\u0646\\u0643 \\u0623\\u0646\\u062a \\u0641\\u064a\\u0647 \\u0645\\u0648\\u062c\\u0648\\u062f \\u0645\\u0633\\u0627\\u0626\\u064a \\u0627\\u0646\\u062a\\u0650 \\u2764\\ufe0f\\ud83d\\udc8b\\ud83c\\udf39\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2740802604,\"id_str\":\"2740802604\",\"name\":\"\\u0631\\u062c\\u0644 \\u0645\\u0632\\u0622\\u062c\\u064a\",\"screen_name\":\"V8888S\",\"location\":\"\",\"url\":null,\"description\":\"\\u0627\\u0646\\u0627 \\u0631\\u062c\\u0644 \\u0645\\u0632\\u0627\\u062c\\u064a \\u0630\\u0622\\u062a \\u0645\\u0641\\u0631\\u062f\\u0622\\u062a \\u0639\\u0646
\\u064a\\u062f\\u06be\\u06c1\\u064e \\u0644\\u0633\\u062a \\u0645\\u062c\\u0628\\u0648\\u0631 \\u0639\\u0644\\u0649 \\u062a\\u0628\\u0631\\u064a\\u064e\\u0631 \\u0645\\u0648\\u0622\\u0642\\u0641\\u064a \\u0644\\u0645\\u0646\\u064e \\u064a\\u0633\\u064a\\u0621 \\u0622\\u0644\\u0638\\u0646 \\u0622\\u062a\\u062d\\u062f\\u062b \\u0628\\u0639\\u0641\\u0648\\u064a\\u06be\\u06c1\\u064e \\u0622\\u0636\\u062d\\u0643\\u0643 \\u0628\\u0634\\u062f\\u06be\\u06c1\\u064e \\u0623\\u063a\\u0627\\u0631 \\u0628\\u062c\\u0646\\u0648\\u0646 \\u0648\\u0623\\u0639\\u0634\\u0642 \\u0628\\u062e\\u062c\\u0644 .. \\u0623\\u0646\\u062a\\u0645\\u064a \\u0644\\u0644\\u0647\\u0644\\u0627\\u0644\",\"protected\":false,\"verified\":false,\"followers_count\":9443,\"friends_count\":153,\"listed_count\":0,\"favourites_count\":81,\"statuses_count\":3183,\"created_at\":\"Mon Aug 18 00:21:00 +0000 2014\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"ar\",\"contributors_enabled\":false,\"is_transl
ator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/501402712164933633\\/jfgUf6cZ_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/501402712164933633\\/jfgUf6cZ_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/2740802604\\/1412092338\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contrib
utors\":null,\"retweet_count\":671,\"favorite_count\":1,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"ar\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"V8888S\",\"name\":\"\\u0631\\u062c\\u0644 \\u0645\\u0632\\u0622\\u062c\\u064a\",\"id\":2740802604,\"id_str\":\"2740802604\",\"indices\":[3,10]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"ar\",\"timestamp_ms\":\"1412178008681\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180891443201,\"id_str\":\"517338180891443201\",\"text\":\"\\u0622\\u0628\\u0644\\u063a \\u0645\\u0646 \\u0627\\u0644\\u0643\\u0644\\u0627\\u0645 [ \\u0627\\u0644\\u0644\\u0647\\u0645 \\u0623\\u0646\\u0639\\u0645 \\u0639\\u0644\\u064a\\u0646\\u0627 \\u0628\\u062d\\u0645\\u062f\\u0643 \\u0648 \\u0631\\u0636\\u0627\\u0643 ] #\\u063a\\u0631\\u062f_\\u0628\\u0635\\u0648\\u0631\\u0629\",\"source\":\"\\u003ca href=\\\"http:\\/\\/batross.com\\\" rel=\\\"nofollow\\\"\\u003e\\u0628\\u0627\\u0644\\u062e\\u064a\\u0631 \\u063a\\u0631\\u062f\\u0644\\u064a\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":603454115,\"id_str\":\"603454115\",\"name\":\"\\u0627\\u0644\\u0643\\u0627\\u0633\\u06312006\",\"screen_name\":\"DKL2006\",\"location\":\"\\u0627\\u0644\\u0631\\u0
64a\\u0627\\u0636\",\"url\":null,\"description\":\"\\u200f\\u200f\\u0645\\u0648\\u0638\\u0641 (\\u0627\\u062d\\u0628 \\u0627\\u0644\\u062d\\u064a\\u0627\\u0629 \\u0648\\u0627\\u0646 \\u063a\\u062f\\u0631\\u0648 \\u0628\\u064a \\u0627\\u0644\\u0628\\u0634\\u0631)\",\"protected\":false,\"verified\":false,\"followers_count\":864,\"friends_count\":1997,\"listed_count\":0,\"favourites_count\":21,\"statuses_count\":5489,\"created_at\":\"Sat Jun 09 09:21:16 +0000 2012\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":true,\"lang\":\"ar\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",
\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/514354297622892545\\/wnTPiDj6_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/514354297622892545\\/wnTPiDj6_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/603454115\\/1411467703\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"\\u063a\\u0631\\u062f_\\u0628\\u0635\\u0648\\u0631\\u0629\",\"indices\":[49,59]}],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"ar\",\"timestamp_ms\":\"1412178008664\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180912443393,\"id_str\":\"517338180912443393\",\"text\":\"@jackspotlight amo os vil\\u00f5es, eles s\\u00e3o mais bonitos e gostosos hhahahhhaa\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/android\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Android\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":517337999437078529,\"in_reply_to_status_id_str\":\"517337999437078529\",\"in_reply_to_user_id\":109532411,\"in_reply_to_user_id_str\":\"109532411\",\"in_reply_to_screen_name\":\"jackspotlight\",\"user\":{\"id\":872394073,\"id_str\":\"872394073\",\"name\":\"styles\",\"screen_name\":\"kylizstyles\",\"location\":\"coxa do harry\",\"url\":\"http:\\/\\/nobody-knows-you-the-way-i-do.tumblr.com\",\"description\":\"jenner's\\/ kardashian's e viados \\u2661 wwa tour \\u2661\",\"protected\":false,\"verified\":false,\"followers_count\":1834,\"friends_count\":1979,\"listed_count\":1,\"favourites_
count\":3815,\"statuses_count\":60978,\"created_at\":\"Wed Oct 10 19:39:49 +0000 2012\",\"utc_offset\":-7200,\"time_zone\":\"Mid-Atlantic\",\"geo_enabled\":true,\"lang\":\"pt\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"FFFFFF\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/469603339643015168\\/scY2ldKq.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/469603339643015168\\/scY2ldKq.jpeg\",\"profile_background_tile\":false,\"profile_link_color\":\"ABB8C2\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/513517169775501312\\/1Emb0kk2_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/513517169775501312\\/1Emb0kk2_normal.jpeg\",\"profi
le_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/872394073\\/1411510767\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"jackspotlight\",\"name\":\"camila\",\"id\":109532411,\"id_str\":\"109532411\",\"indices\":[0,14]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"pt\",\"timestamp_ms\":\"1412178008671\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180895649793,\"id_str\":\"517338180895649793\",\"text\":\"Fat chats - The good, the bad and the ugly comments: Study analyzes how people chat about weight on different socia\\u2026 http:\\/\\/t.co\\/kIOa0ebpn7\",\"source\":\"\\u003ca href=\\\"http:\\/\\/www.reddit.com\\/\\\" rel=\\\"nofollow\\\"\\u003eReddit RSS\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2577635288,\"id_str\":\"2577635288\",\"name\":\"Reddit Science\",\"screen_name\":\"Science_Reddit\",\"location\":\"\",\"url\":\"http:\\/\\/www.reddit.com\\/r\\/science\\/new\\/\",\"description\":\"Newest submissions from \\/r\\/science with original links.\",\"protected\":false,\"verified\":false,\"followers_count\":272,\"friends_count\":12,\"listed_count\":6,\"favourites_count\":0,\"statuses_c
ount\":9549,\"created_at\":\"Thu Jun 19 23:11:01 +0000 2014\",\"utc_offset\":10800,\"time_zone\":\"Tallinn\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/480229204919742465\\/DHLw_QAc_normal.png\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/480229204919742465\\/DHLw_QAc_normal.png\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent
\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[{\"url\":\"http:\\/\\/t.co\\/kIOa0ebpn7\",\"expanded_url\":\"http:\\/\\/www.springer.com\\/gp\\/about-springer\\/media\\/springer-select\\/fat-chats-the-good-the-bad-and-the-ugly-comments\\/35816\",\"display_url\":\"springer.com\\/gp\\/about-sprin\\u2026\",\"indices\":[117,139]}],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_ms\":\"1412178008665\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180908249088,\"id_str\":\"517338180908249088\",\"text\":\"When you truly forgive if ever given the opportunity to get even you won't even new worried about it you just move on\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/android\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Android\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":268692270,\"id_str\":\"268692270\",\"name\":\"Mz. Bossy \",\"screen_name\":\"MzBarbieondat\",\"location\":\"Da Pharmacy \",\"url\":null,\"description\":\"I am that bitch you either love to hate or hate to love either way I'm still dat bitch! #ObeyReality #UGLYGang #CaponeNation #WeMoBBin\",\"protected\":false,\"verified\":false,\"followers_count\":1386,\"friends_count\":1990,\"listed_count\":1,\"favourit
es_count\":457,\"statuses_count\":9563,\"created_at\":\"Sat Mar 19 08:18:08 +0000 2011\",\"utc_offset\":-18000,\"time_zone\":\"Central Time (US & Canada)\",\"geo_enabled\":true,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/730188034\\/989425a06f0c5e65ce6b68c77c2551a1.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/730188034\\/989425a06f0c5e65ce6b68c77c2551a1.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/465219107516723200\\/DHqepczu_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/465219
107516723200\\/DHqepczu_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/268692270\\/1411370160\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":{\"type\":\"Point\",\"coordinates\":[34.923717,-88.496280]},\"coordinates\":{\"type\":\"Point\",\"coordinates\":[-88.496280,34.923717]},\"place\":{\"id\":\"8c00285181cdb467\",\"url\":\"https:\\/\\/api.twitter.com\\/1.1\\/geo\\/id\\/8c00285181cdb467.json\",\"place_type\":\"city\",\"name\":\"Corinth\",\"full_name\":\"Corinth, MS\",\"country_code\":\"US\",\"country\":\"United States\",\"bounding_box\":{\"type\":\"Polygon\",\"coordinates\":[[[-88.576522,34.901195],[-88.576522,34.964867],[-88.4694382,34.964867],[-88.4694382,34.901195]]]},\"attributes\":{}},\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited
\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_ms\":\"1412178008669\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180882690049,\"id_str\":\"517338180882690049\",\"text\":\"I complimented Susie McBeth\\u2019s page at @aboutdotme! Check it out: http:\\/\\/t.co\\/KYa7ly9oyo @susiemcbeth\",\"source\":\"\\u003ca href=\\\"https:\\/\\/dev.twitter.com\\/docs\\/tfw\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Websites\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2331974365,\"id_str\":\"2331974365\",\"name\":\"Miranda N. Prather\",\"screen_name\":\"MirandaNPrather\",\"location\":\"Middletown, MD\",\"url\":\"http:\\/\\/www.facebook.com\\/blueblueseaottb\",\"description\":\"Writer, horselover, Born in TX, but got my soul in NOLA\\r\\n\\r\\nArea Coordinator for: http:\\/\\/www.grantstoyou.org\\/chapterdisplay.php?id=104\",\"protected\":false,\"verified\":false,\"followers_
count\":94,\"friends_count\":266,\"listed_count\":2,\"favourites_count\":481,\"statuses_count\":616,\"created_at\":\"Fri Feb 07 14:44:57 +0000 2014\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/512644583595835392\\/U80V8y5D.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/512644583595835392\\/U80V8y5D.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"333333\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"0084B4\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/512645616459980800\\/ZOD6qc7y_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/51
2645616459980800\\/ZOD6qc7y_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/2331974365\\/1411059107\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[{\"url\":\"http:\\/\\/t.co\\/KYa7ly9oyo\",\"expanded_url\":\"http:\\/\\/about.me\\/susiemcbeth\",\"display_url\":\"about.me\\/susiemcbeth\",\"indices\":[65,87]}],\"user_mentions\":[{\"screen_name\":\"aboutdotme\",\"name\":\"about.me\",\"id\":115304519,\"id_str\":\"115304519\",\"indices\":[38,49]},{\"screen_name\":\"susiemcbeth\",\"name\":\"Susie McBeth\",\"id\":22052388,\"id_str\":\"22052388\",\"indices\":[88,100]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_m
s\":\"1412178008664\"}",
+ "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180904026113,\"id_str\":\"517338180904026113\",\"text\":\"RT @danmfc90: \\u201c@theawayfans: Roma fans celebrating their goal at The Etihad last night http:\\/\\/t.co\\/XrRhhiQE0A\\u201d lad in the Millwall hat ther\\u2026\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":473835254,\"id_str\":\"473835254\",\"name\":\"harry weaver\",\"screen_name\":\"1HarryWeaver1\",\"location\":\"\",\"url\":null,\"description\":null,\"protected\":false,\"verified\":false,\"followers_count\":490,\"friends_count\":1542,\"listed_count\":0,\"favourites_count\":176,\"statuses_count\":2109,\"created_at\":\"Wed Jan 25 11:28:00 +0000 2012\",\"utc_offse
t\":null,\"time_zone\":null,\"geo_enabled\":true,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/486841189346582529\\/VFZXNXYx_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/486841189346582529\\/VFZXNXYx_normal.jpeg\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null
,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 12:55:49 +0000 2014\",\"id\":517296830582108161,\"id_str\":\"517296830582108161\",\"text\":\"\\u201c@theawayfans: Roma fans celebrating their goal at The Etihad last night http:\\/\\/t.co\\/XrRhhiQE0A\\u201d lad in the Millwall hat there haha \\ud83d\\udc4f\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":517287910224461824,\"in_reply_to_status_id_str\":\"517287910224461824\",\"in_reply_to_user_id\":1107326988,\"in_reply_to_user_id_str\":\"1107326988\",\"in_reply_to_screen_name\":\"theawayfans\",\"user\":{\"id\":258893426,\"id_str\":\"258893426\",\"name\":\"Dan Pullen\",\"screen_name\":\"danmfc90\",\"location\":\"Essex\",\"url\":null,\"description\":\"Im an opinionated baby eating wife stabbing millwall fan who enjoys long drunk train journeys up north.. Will probably of
fend you tbh... Dont believe the hype\",\"protected\":false,\"verified\":false,\"followers_count\":1537,\"friends_count\":1325,\"listed_count\":6,\"favourites_count\":17583,\"statuses_count\":55539,\"created_at\":\"Mon Feb 28 18:48:57 +0000 2011\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":true,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/512530957979881473\\/waAvdqKb_normal.jpeg\",\"profile_image_url_https\":\"https:\
\/\\/pbs.twimg.com\\/profile_images\\/512530957979881473\\/waAvdqKb_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/258893426\\/1408263439\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":4,\"favorite_count\":4,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"theawayfans\",\"name\":\"The Away Fans\",\"id\":1107326988,\"id_str\":\"1107326988\",\"indices\":[1,13]}],\"symbols\":[],\"media\":[{\"id\":517287908084948992,\"id_str\":\"517287908084948992\",\"indices\":[73,95],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3GnJgCIAAK2O8.jpg\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3GnJgCIAAK2O8.jpg\",\"url\":\"http:\\/\\/t.co\\/XrRhhiQE0A\",\"display_url\":\"pic.twitter.com\\/XrRhhiQE0A\",\"expanded_url\":\"http:\\/\
\/twitter.com\\/theawayfans\\/status\\/517287910224461824\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"large\":{\"w\":952,\"h\":594,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":212,\"resize\":\"fit\"},\"medium\":{\"w\":600,\"h\":374,\"resize\":\"fit\"}},\"source_status_id\":517287910224461824,\"source_status_id_str\":\"517287910224461824\"}]},\"extended_entities\":{\"media\":[{\"id\":517287908084948992,\"id_str\":\"517287908084948992\",\"indices\":[73,95],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3GnJgCIAAK2O8.jpg\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3GnJgCIAAK2O8.jpg\",\"url\":\"http:\\/\\/t.co\\/XrRhhiQE0A\",\"display_url\":\"pic.twitter.com\\/XrRhhiQE0A\",\"expanded_url\":\"http:\\/\\/twitter.com\\/theawayfans\\/status\\/517287910224461824\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"large\":{\"w\":952,\"h\":594,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"smal
l\":{\"w\":340,\"h\":212,\"resize\":\"fit\"},\"medium\":{\"w\":600,\"h\":374,\"resize\":\"fit\"}},\"source_status_id\":51728791022446182
<TRUNCATED>
[24/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceCombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceCombineFunction.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceCombineFunction.java
new file mode 100644
index 0000000..aa9f048
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceCombineFunction.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapred;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.RichGroupReduceFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopOutputCollector;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopTupleUnwrappingIterator;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * This wrapper maps a Hadoop Reducer and Combiner (mapred API) to a combinable Flink GroupReduceFunction.
+ */
+@SuppressWarnings("rawtypes")
+@org.apache.flink.api.common.functions.RichGroupReduceFunction.Combinable
+public final class HadoopReduceCombineFunction<KEYIN, VALUEIN, KEYOUT, VALUEOUT>
+ extends RichGroupReduceFunction<Tuple2<KEYIN,VALUEIN>,Tuple2<KEYOUT,VALUEOUT>>
+ implements ResultTypeQueryable<Tuple2<KEYOUT,VALUEOUT>>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private transient Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT> reducer;
+ private transient Reducer<KEYIN,VALUEIN,KEYIN,VALUEIN> combiner;
+ private transient JobConf jobConf;
+
+ private transient HadoopTupleUnwrappingIterator<KEYIN, VALUEIN> valueIterator;
+ private transient HadoopOutputCollector<KEYOUT,VALUEOUT> reduceCollector;
+ private transient HadoopOutputCollector<KEYIN,VALUEIN> combineCollector;
+ private transient Reporter reporter;
+
+ /**
+ * Maps two Hadoop Reducer (mapred API) to a combinable Flink GroupReduceFunction.
+ *
+ * @param hadoopReducer The Hadoop Reducer that is mapped to a GroupReduceFunction.
+ * @param hadoopCombiner The Hadoop Reducer that is mapped to the combiner function.
+ */
+ public HadoopReduceCombineFunction(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopReducer,
+ Reducer<KEYIN,VALUEIN,KEYIN,VALUEIN> hadoopCombiner) {
+ this(hadoopReducer, hadoopCombiner, new JobConf());
+ }
+
+ /**
+ * Maps two Hadoop Reducer (mapred API) to a combinable Flink GroupReduceFunction.
+ *
+ * @param hadoopReducer The Hadoop Reducer that is mapped to a GroupReduceFunction.
+ * @param hadoopCombiner The Hadoop Reducer that is mapped to the combiner function.
+ * @param conf The JobConf that is used to configure both Hadoop Reducers.
+ */
+ public HadoopReduceCombineFunction(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopReducer,
+ Reducer<KEYIN,VALUEIN,KEYIN,VALUEIN> hadoopCombiner, JobConf conf) {
+ if(hadoopReducer == null) {
+ throw new NullPointerException("Reducer may not be null.");
+ }
+ if(hadoopCombiner == null) {
+ throw new NullPointerException("Combiner may not be null.");
+ }
+ if(conf == null) {
+ throw new NullPointerException("JobConf may not be null.");
+ }
+
+ this.reducer = hadoopReducer;
+ this.combiner = hadoopCombiner;
+ this.jobConf = conf;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ this.reducer.configure(jobConf);
+ this.combiner.configure(jobConf);
+
+ this.reporter = new HadoopDummyReporter();
+ Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
+ this.valueIterator = new HadoopTupleUnwrappingIterator<KEYIN, VALUEIN>(inKeyClass);
+ this.combineCollector = new HadoopOutputCollector<KEYIN, VALUEIN>();
+ this.reduceCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
+ }
+
+ @Override
+ public void reduce(final Iterable<Tuple2<KEYIN,VALUEIN>> values, final Collector<Tuple2<KEYOUT,VALUEOUT>> out)
+ throws Exception {
+ reduceCollector.setFlinkCollector(out);
+ valueIterator.set(values.iterator());
+ reducer.reduce(valueIterator.getCurrentKey(), valueIterator, reduceCollector, reporter);
+ }
+
+ @Override
+ public void combine(final Iterable<Tuple2<KEYIN,VALUEIN>> values, final Collector<Tuple2<KEYIN,VALUEIN>> out) throws Exception {
+ combineCollector.setFlinkCollector(out);
+ valueIterator.set(values.iterator());
+ combiner.reduce(valueIterator.getCurrentKey(), valueIterator, combineCollector, reporter);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public TypeInformation<Tuple2<KEYOUT,VALUEOUT>> getProducedType() {
+ Class<KEYOUT> outKeyClass = (Class<KEYOUT>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 2);
+ Class<VALUEOUT> outValClass = (Class<VALUEOUT>)TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 3);
+
+ final TypeInformation<KEYOUT> keyTypeInfo = TypeExtractor.getForClass((Class<KEYOUT>) outKeyClass);
+ final TypeInformation<VALUEOUT> valueTypleInfo = TypeExtractor.getForClass((Class<VALUEOUT>) outValClass);
+ return new TupleTypeInfo<Tuple2<KEYOUT,VALUEOUT>>(keyTypeInfo, valueTypleInfo);
+ }
+
+ /**
+ * Custom serialization methods.
+ * @see http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html
+ */
+ private void writeObject(final ObjectOutputStream out) throws IOException {
+
+ out.writeObject(reducer.getClass());
+ out.writeObject(combiner.getClass());
+ jobConf.write(out);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
+
+ Class<Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT>> reducerClass =
+ (Class<Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT>>)in.readObject();
+ reducer = InstantiationUtil.instantiate(reducerClass);
+
+ Class<Reducer<KEYIN,VALUEIN,KEYIN,VALUEIN>> combinerClass =
+ (Class<Reducer<KEYIN,VALUEIN,KEYIN,VALUEIN>>)in.readObject();
+ combiner = InstantiationUtil.instantiate(combinerClass);
+
+ jobConf = new JobConf();
+ jobConf.readFields(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceFunction.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceFunction.java
new file mode 100644
index 0000000..d9797c3
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceFunction.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapred;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.RichGroupReduceFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopOutputCollector;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopTupleUnwrappingIterator;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * This wrapper maps a Hadoop Reducer (mapred API) to a non-combinable Flink GroupReduceFunction.
+ */
+@SuppressWarnings("rawtypes")
+public final class HadoopReduceFunction<KEYIN, VALUEIN, KEYOUT, VALUEOUT>
+ extends RichGroupReduceFunction<Tuple2<KEYIN,VALUEIN>,Tuple2<KEYOUT,VALUEOUT>>
+ implements ResultTypeQueryable<Tuple2<KEYOUT,VALUEOUT>>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private transient Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT> reducer;
+ private transient JobConf jobConf;
+
+ private transient HadoopTupleUnwrappingIterator<KEYIN, VALUEIN> valueIterator;
+ private transient HadoopOutputCollector<KEYOUT,VALUEOUT> reduceCollector;
+ private transient Reporter reporter;
+
+ /**
+ * Maps a Hadoop Reducer (mapred API) to a non-combinable Flink GroupReduceFunction.
+ *
+ * @param hadoopReducer The Hadoop Reducer to wrap.
+ */
+ public HadoopReduceFunction(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopReducer) {
+ this(hadoopReducer, new JobConf());
+ }
+
+ /**
+ * Maps a Hadoop Reducer (mapred API) to a non-combinable Flink GroupReduceFunction.
+ *
+ * @param hadoopReducer The Hadoop Reducer to wrap.
+ * @param conf The JobConf that is used to configure the Hadoop Reducer.
+ */
+ public HadoopReduceFunction(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopReducer, JobConf conf) {
+ if(hadoopReducer == null) {
+ throw new NullPointerException("Reducer may not be null.");
+ }
+ if(conf == null) {
+ throw new NullPointerException("JobConf may not be null.");
+ }
+
+ this.reducer = hadoopReducer;
+ this.jobConf = conf;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ this.reducer.configure(jobConf);
+
+ this.reporter = new HadoopDummyReporter();
+ this.reduceCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
+ Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
+ this.valueIterator = new HadoopTupleUnwrappingIterator<KEYIN, VALUEIN>(inKeyClass);
+ }
+
+ @Override
+ public void reduce(final Iterable<Tuple2<KEYIN,VALUEIN>> values, final Collector<Tuple2<KEYOUT,VALUEOUT>> out)
+ throws Exception {
+
+ reduceCollector.setFlinkCollector(out);
+ valueIterator.set(values.iterator());
+ reducer.reduce(valueIterator.getCurrentKey(), valueIterator, reduceCollector, reporter);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public TypeInformation<Tuple2<KEYOUT,VALUEOUT>> getProducedType() {
+ Class<KEYOUT> outKeyClass = (Class<KEYOUT>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 2);
+ Class<VALUEOUT> outValClass = (Class<VALUEOUT>)TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 3);
+
+ final TypeInformation<KEYOUT> keyTypeInfo = TypeExtractor.getForClass((Class<KEYOUT>) outKeyClass);
+ final TypeInformation<VALUEOUT> valueTypleInfo = TypeExtractor.getForClass((Class<VALUEOUT>) outValClass);
+ return new TupleTypeInfo<Tuple2<KEYOUT,VALUEOUT>>(keyTypeInfo, valueTypleInfo);
+ }
+
+ /**
+ * Custom serialization methods.
+ * @see http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html
+ */
+ private void writeObject(final ObjectOutputStream out) throws IOException {
+
+ out.writeObject(reducer.getClass());
+ jobConf.write(out);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
+
+ Class<Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT>> reducerClass =
+ (Class<Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT>>)in.readObject();
+ reducer = InstantiationUtil.instantiate(reducerClass);
+
+ jobConf = new JobConf();
+ jobConf.readFields(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/HadoopMapredCompatWordCount.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/HadoopMapredCompatWordCount.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/HadoopMapredCompatWordCount.java
new file mode 100644
index 0000000..de20fab
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/HadoopMapredCompatWordCount.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapred.example;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.hadoopcompatibility.mapred.HadoopInputFormat;
+import org.apache.flink.hadoopcompatibility.mapred.HadoopMapFunction;
+import org.apache.flink.hadoopcompatibility.mapred.HadoopOutputFormat;
+import org.apache.flink.hadoopcompatibility.mapred.HadoopReduceCombineFunction;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+
+
+
+/**
+ * Implements a word count which takes the input file and counts the number of
+ * occurrences of each word in the file and writes the result back to disk.
+ *
+ * This example shows how to use Hadoop Input Formats, how to convert Hadoop Writables to
+ * common Java types for better usage in a Flink job and how to use Hadoop Output Formats.
+ */
+public class HadoopMapredCompatWordCount {
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 2) {
+ System.err.println("Usage: WordCount <input path> <result path>");
+ return;
+ }
+
+ final String inputPath = args[0];
+ final String outputPath = args[1];
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ // Set up the Hadoop Input Format
+ HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, new JobConf());
+ TextInputFormat.addInputPath(hadoopInputFormat.getJobConf(), new Path(inputPath));
+
+ // Create a Flink job with it
+ DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);
+
+ DataSet<Tuple2<Text, LongWritable>> words =
+ text.flatMap(new HadoopMapFunction<LongWritable, Text, Text, LongWritable>(new Tokenizer()))
+ .groupBy(0).reduceGroup(new HadoopReduceCombineFunction<Text, LongWritable, Text, LongWritable>(new Counter(), new Counter()));
+
+ // Set up Hadoop Output Format
+ HadoopOutputFormat<Text, LongWritable> hadoopOutputFormat =
+ new HadoopOutputFormat<Text, LongWritable>(new TextOutputFormat<Text, LongWritable>(), new JobConf());
+ hadoopOutputFormat.getJobConf().set("mapred.textoutputformat.separator", " ");
+ TextOutputFormat.setOutputPath(hadoopOutputFormat.getJobConf(), new Path(outputPath));
+
+ // Output & Execute
+ words.output(hadoopOutputFormat).setParallelism(1);
+ env.execute("Hadoop Compat WordCount");
+ }
+
+
+ public static final class Tokenizer implements Mapper<LongWritable, Text, Text, LongWritable> {
+
+ @Override
+ public void map(LongWritable k, Text v, OutputCollector<Text, LongWritable> out, Reporter rep)
+ throws IOException {
+ // normalize and split the line
+ String line = v.toString();
+ String[] tokens = line.toLowerCase().split("\\W+");
+
+ // emit the pairs
+ for (String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new Text(token), new LongWritable(1l));
+ }
+ }
+ }
+
+ @Override
+ public void configure(JobConf arg0) { }
+
+ @Override
+ public void close() throws IOException { }
+
+ }
+
+ public static final class Counter implements Reducer<Text, LongWritable, Text, LongWritable> {
+
+ @Override
+ public void reduce(Text k, Iterator<LongWritable> vs, OutputCollector<Text, LongWritable> out, Reporter rep)
+ throws IOException {
+
+ long cnt = 0;
+ while(vs.hasNext()) {
+ cnt += vs.next().get();
+ }
+ out.collect(k, new LongWritable(cnt));
+
+ }
+
+ @Override
+ public void configure(JobConf arg0) { }
+
+ @Override
+ public void close() throws IOException { }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
new file mode 100644
index 0000000..ff28a59
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.java.record.operators.GenericDataSink;
+import org.apache.flink.compiler.contextcheck.Validatable;
+import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.DefaultFlinkTypeConverter;
+import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.FlinkTypeConverter;
+import org.apache.flink.types.Record;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputFormat;
+
+/**
+ * The HadoopDataSink is a generic wrapper for all Hadoop OutputFormats.
+ *
+ * Example usage:
+ * <pre>
+ * HadoopDataSink out = new HadoopDataSink(new org.apache.hadoop.mapred.TextOutputFormat<Text, IntWritable>(), new JobConf(), "Hadoop TextOutputFormat",reducer, Text.class,IntWritable.class);
+ * org.apache.hadoop.mapred.TextOutputFormat.setOutputPath(out.getJobConf(), new Path(output));
+ * </pre>
+ *
+ * Note that it is possible to provide custom data type converter.
+ *
+ * The HadoopDataSink provides a default converter: {@link org.apache.flink.hadoopcompatibility.mapred.record.datatypes.DefaultFlinkTypeConverter}
+ **/
+public class HadoopDataSink<K,V> extends GenericDataSink implements Validatable {
+
+ private static String DEFAULT_NAME = "<Unnamed Hadoop Data Sink>";
+
+ private JobConf jobConf;
+
+ public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, Operator<Record> input, FlinkTypeConverter<K,V> conv, Class<K> keyClass, Class<V> valueClass) {
+ this(hadoopFormat, jobConf, name, Collections.<Operator<Record>>singletonList(input), conv, keyClass, valueClass);
+ }
+
+ public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, Operator<Record> input, Class<K> keyClass, Class<V> valueClass) {
+ this(hadoopFormat, jobConf, name, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
+ }
+
+ public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, Operator<Record> input, Class<K> keyClass, Class<V> valueClass) {
+ this(hadoopFormat, jobConf, DEFAULT_NAME, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
+ }
+
+ public HadoopDataSink(OutputFormat<K,V> hadoopFormat, Operator<Record> input, Class<K> keyClass, Class<V> valueClass) {
+ this(hadoopFormat, new JobConf(), DEFAULT_NAME, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
+ }
+
+
+
+ @SuppressWarnings("deprecation")
+ public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, List<Operator<Record>> input, FlinkTypeConverter<K,V> conv, Class<K> keyClass, Class<V> valueClass) {
+ super(new HadoopRecordOutputFormat<K,V>(hadoopFormat, jobConf, conv),input, name);
+
+ if (hadoopFormat == null || jobConf == null) {
+ throw new NullPointerException();
+ }
+
+ this.name = name;
+ this.jobConf = jobConf;
+ jobConf.setOutputKeyClass(keyClass);
+ jobConf.setOutputValueClass(valueClass);
+ }
+
+ public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, List<Operator<Record>> input, Class<K> keyClass, Class<V> valueClass) {
+ this(hadoopFormat, jobConf, name, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
+ }
+
+ public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, List<Operator<Record>> input, Class<K> keyClass, Class<V> valueClass) {
+ this(hadoopFormat, jobConf, DEFAULT_NAME, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
+ }
+
+ public HadoopDataSink(OutputFormat<K,V> hadoopFormat, List<Operator<Record>> input, Class<K> keyClass, Class<V> valueClass) {
+ this(hadoopFormat, new JobConf(), DEFAULT_NAME, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
+ }
+
+ public JobConf getJobConf() {
+ return this.jobConf;
+ }
+
+ @Override
+ public void check() {
+ // see for more details https://github.com/stratosphere/stratosphere/pull/531
+ if (FileOutputFormat.getOutputPath(jobConf) == null) {
+ throw new NullPointerException("The HadoopDataSink currently expects a correct outputPath.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSource.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSource.java
new file mode 100644
index 0000000..508f069
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSource.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapred.record;
+
+
+import org.apache.flink.api.java.record.operators.GenericDataSource;
+import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.DefaultHadoopTypeConverter;
+import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.HadoopTypeConverter;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobConf;
+
+/**
+ * The HadoopDataSource is a generic wrapper for all Hadoop InputFormats.
+ *
+ * Example usage:
+ * <pre>
+ * HadoopDataSource source = new HadoopDataSource(new org.apache.hadoop.mapred.TextInputFormat(), new JobConf(), "Input Lines");
+ * org.apache.hadoop.mapred.TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
+ * </pre>
+ *
+ * Note that it is possible to provide custom data type converter.
+ *
+ * The HadoopDataSource provides two different standard converters:
+ * * WritableWrapperConverter: Converts Hadoop Types to a record that contains a WritableComparableWrapper (key) and a WritableWrapper
+ * * DefaultHadoopTypeConverter: Converts the standard hadoop types (longWritable, Text) to Flinks's {@link org.apache.flink.types.Value} types.
+ */
+public class HadoopDataSource<K,V> extends GenericDataSource<HadoopRecordInputFormat<K,V>> {
+
+ private static String DEFAULT_NAME = "<Unnamed Hadoop Data Source>";
+
+ private JobConf jobConf;
+
+ /**
+ *
+ * @param hadoopFormat Implementation of a Hadoop input format
+ * @param jobConf JobConf object (Hadoop)
+ * @param name Name of the DataSource
+ * @param conv Definition of a custom type converter {@link DefaultHadoopTypeConverter}.
+ */
+ public HadoopDataSource(InputFormat<K,V> hadoopFormat, JobConf jobConf, String name, HadoopTypeConverter<K,V> conv) {
+ super(new HadoopRecordInputFormat<K,V>(hadoopFormat, jobConf, conv),name);
+
+ if (hadoopFormat == null || jobConf == null || conv == null) {
+ throw new NullPointerException();
+ }
+
+ this.name = name;
+ this.jobConf = jobConf;
+ }
+
+ public HadoopDataSource(InputFormat<K,V> hadoopFormat, JobConf jobConf, String name) {
+ this(hadoopFormat, jobConf, name, new DefaultHadoopTypeConverter<K,V>() );
+ }
+ public HadoopDataSource(InputFormat<K,V> hadoopFormat, JobConf jobConf) {
+ this(hadoopFormat, jobConf, DEFAULT_NAME);
+ }
+
+ public HadoopDataSource(InputFormat<K,V> hadoopFormat) {
+ this(hadoopFormat, new JobConf(), DEFAULT_NAME);
+ }
+
+ public JobConf getJobConf() {
+ return this.jobConf;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordInputFormat.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordInputFormat.java
new file mode 100644
index 0000000..275fd4c
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordInputFormat.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+import org.apache.flink.api.common.io.DefaultInputSplitAssigner;
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.io.statistics.BaseStatistics;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.io.InputSplitAssigner;
+import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.HadoopTypeConverter;
+import org.apache.flink.hadoopcompatibility.mapred.utils.HadoopUtils;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopInputSplit;
+import org.apache.flink.types.Record;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.util.ReflectionUtils;
+
+public class HadoopRecordInputFormat<K, V> implements InputFormat<Record, HadoopInputSplit> {
+
+ private static final long serialVersionUID = 1L;
+
+ public org.apache.hadoop.mapred.InputFormat<K, V> hadoopInputFormat;
+ public HadoopTypeConverter<K,V> converter;
+ private String hadoopInputFormatName;
+ public JobConf jobConf;
+ public transient K key;
+ public transient V value;
+ public RecordReader<K, V> recordReader;
+ private boolean fetched = false;
+ private boolean hasNext;
+
+ public HadoopRecordInputFormat() {
+ super();
+ }
+
+ public HadoopRecordInputFormat(org.apache.hadoop.mapred.InputFormat<K,V> hadoopInputFormat, JobConf job, HadoopTypeConverter<K,V> conv) {
+ super();
+ this.hadoopInputFormat = hadoopInputFormat;
+ this.hadoopInputFormatName = hadoopInputFormat.getClass().getName();
+ this.converter = conv;
+ HadoopUtils.mergeHadoopConf(job);
+ this.jobConf = job;
+ }
+
+ @Override
+ public void configure(Configuration parameters) {
+
+ }
+
+ @Override
+ public BaseStatistics getStatistics(BaseStatistics cachedStatistics) throws IOException {
+ return null;
+ }
+
+ @Override
+ public HadoopInputSplit[] createInputSplits(int minNumSplits)
+ throws IOException {
+ org.apache.hadoop.mapred.InputSplit[] splitArray = hadoopInputFormat.getSplits(jobConf, minNumSplits);
+ HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
+ for(int i=0;i<splitArray.length;i++){
+ hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
+ }
+ return hiSplit;
+ }
+
+ @Override
+ public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
+ return new DefaultInputSplitAssigner(inputSplits);
+ }
+
+ @Override
+ public void open(HadoopInputSplit split) throws IOException {
+ this.recordReader = this.hadoopInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
+ key = this.recordReader.createKey();
+ value = this.recordReader.createValue();
+ this.fetched = false;
+ }
+
+ private void fetchNext() throws IOException {
+ hasNext = this.recordReader.next(key, value);
+ fetched = true;
+ }
+
+ @Override
+ public boolean reachedEnd() throws IOException {
+ if(!fetched) {
+ fetchNext();
+ }
+ return !hasNext;
+ }
+
+ @Override
+ public Record nextRecord(Record record) throws IOException {
+ if(!fetched) {
+ fetchNext();
+ }
+ if(!hasNext) {
+ return null;
+ }
+ converter.convert(record, key, value);
+ fetched = false;
+ return record;
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.recordReader.close();
+ }
+
+ /**
+ * Custom serialization methods.
+ * @see "http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html"
+ */
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeUTF(hadoopInputFormatName);
+ jobConf.write(out);
+ out.writeObject(converter);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ hadoopInputFormatName = in.readUTF();
+ if(jobConf == null) {
+ jobConf = new JobConf();
+ }
+ jobConf.readFields(in);
+ try {
+ this.hadoopInputFormat = (org.apache.hadoop.mapred.InputFormat<K,V>) Class.forName(this.hadoopInputFormatName).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to instantiate the hadoop input format", e);
+ }
+ ReflectionUtils.setConf(hadoopInputFormat, jobConf);
+ converter = (HadoopTypeConverter<K,V>) in.readObject();
+ }
+
+ public void setJobConf(JobConf job) {
+ this.jobConf = job;
+ }
+
+
+ public org.apache.hadoop.mapred.InputFormat<K,V> getHadoopInputFormat() {
+ return hadoopInputFormat;
+ }
+
+ public void setHadoopInputFormat(org.apache.hadoop.mapred.InputFormat<K,V> hadoopInputFormat) {
+ this.hadoopInputFormat = hadoopInputFormat;
+ }
+
+ public JobConf getJobConf() {
+ return jobConf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordOutputFormat.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordOutputFormat.java
new file mode 100644
index 0000000..74118a3
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordOutputFormat.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+import org.apache.flink.api.common.io.OutputFormat;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.HadoopFileOutputCommitter;
+import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.FlinkTypeConverter;
+import org.apache.flink.hadoopcompatibility.mapred.utils.HadoopUtils;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyProgressable;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
+import org.apache.flink.types.Record;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.TaskAttemptID;
+import org.apache.hadoop.util.ReflectionUtils;
+
+
+public class HadoopRecordOutputFormat<K,V> implements OutputFormat<Record> {
+
+ private static final long serialVersionUID = 1L;
+
+ public JobConf jobConf;
+
+ public org.apache.hadoop.mapred.OutputFormat<K,V> hadoopOutputFormat;
+
+ private String hadoopOutputFormatName;
+
+ public RecordWriter<K,V> recordWriter;
+
+ public FlinkTypeConverter<K,V> converter;
+
+ public HadoopFileOutputCommitter fileOutputCommitterWrapper;
+
+ public HadoopRecordOutputFormat(org.apache.hadoop.mapred.OutputFormat<K,V> hadoopFormat, JobConf job, FlinkTypeConverter<K,V> conv) {
+ super();
+ this.hadoopOutputFormat = hadoopFormat;
+ this.hadoopOutputFormatName = hadoopFormat.getClass().getName();
+ this.converter = conv;
+ this.fileOutputCommitterWrapper = new HadoopFileOutputCommitter();
+ HadoopUtils.mergeHadoopConf(job);
+ this.jobConf = job;
+ }
+
+ @Override
+ public void configure(Configuration parameters) {
+ }
+
+ /**
+ * create the temporary output file for hadoop RecordWriter.
+ * @param taskNumber The number of the parallel instance.
+ * @param numTasks The number of parallel tasks.
+ * @throws IOException
+ */
+ @Override
+ public void open(int taskNumber, int numTasks) throws IOException {
+ this.fileOutputCommitterWrapper.setupJob(this.jobConf);
+ if (Integer.toString(taskNumber + 1).length() <= 6) {
+ this.jobConf.set("mapred.task.id", "attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s"," ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0");
+ //compatible for hadoop 2.2.0, the temporary output directory is different from hadoop 1.2.1
+ this.jobConf.set("mapreduce.task.output.dir", this.fileOutputCommitterWrapper.getTempTaskOutputPath(this.jobConf,TaskAttemptID.forName(this.jobConf.get("mapred.task.id"))).toString());
+ } else {
+ throw new IOException("task id too large");
+ }
+ this.recordWriter = this.hadoopOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable());
+ }
+
+
+ @Override
+ public void writeRecord(Record record) throws IOException {
+ K key = this.converter.convertKey(record);
+ V value = this.converter.convertValue(record);
+ this.recordWriter.write(key, value);
+ }
+
+ /**
+ * commit the task by moving the output file out from the temporary directory.
+ * @throws IOException
+ */
+ @Override
+ public void close() throws IOException {
+ this.recordWriter.close(new HadoopDummyReporter());
+ if (this.fileOutputCommitterWrapper.needsTaskCommit(this.jobConf, TaskAttemptID.forName(this.jobConf.get("mapred.task.id")))) {
+ this.fileOutputCommitterWrapper.commitTask(this.jobConf, TaskAttemptID.forName(this.jobConf.get("mapred.task.id")));
+ }
+ //TODO: commitjob when all the tasks are finished
+ }
+
+
+ /**
+ * Custom serialization methods.
+ * @see "http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html"
+ */
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeUTF(hadoopOutputFormatName);
+ jobConf.write(out);
+ out.writeObject(converter);
+ out.writeObject(fileOutputCommitterWrapper);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ hadoopOutputFormatName = in.readUTF();
+ if(jobConf == null) {
+ jobConf = new JobConf();
+ }
+ jobConf.readFields(in);
+ try {
+ this.hadoopOutputFormat = (org.apache.hadoop.mapred.OutputFormat<K,V>) Class.forName(this.hadoopOutputFormatName).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to instantiate the hadoop output format", e);
+ }
+ ReflectionUtils.setConf(hadoopOutputFormat, jobConf);
+ converter = (FlinkTypeConverter<K,V>) in.readObject();
+ fileOutputCommitterWrapper = (HadoopFileOutputCommitter) in.readObject();
+ }
+
+
+ public void setJobConf(JobConf job) {
+ this.jobConf = job;
+ }
+
+ public JobConf getJobConf() {
+ return jobConf;
+ }
+
+ public org.apache.hadoop.mapred.OutputFormat<K,V> getHadoopOutputFormat() {
+ return hadoopOutputFormat;
+ }
+
+ public void setHadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K,V> hadoopOutputFormat) {
+ this.hadoopOutputFormat = hadoopOutputFormat;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultFlinkTypeConverter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultFlinkTypeConverter.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultFlinkTypeConverter.java
new file mode 100644
index 0000000..9d37988
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultFlinkTypeConverter.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
+
+import org.apache.flink.types.BooleanValue;
+import org.apache.flink.types.ByteValue;
+import org.apache.flink.types.DoubleValue;
+import org.apache.flink.types.FloatValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.StringValue;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Convert Flink Record into the default hadoop writables.
+ */
+public class DefaultFlinkTypeConverter<K,V> implements FlinkTypeConverter<K,V> {
+ private static final long serialVersionUID = 1L;
+
+ private Class<K> keyClass;
+ private Class<V> valueClass;
+
+ public DefaultFlinkTypeConverter(Class<K> keyClass, Class<V> valueClass) {
+ this.keyClass= keyClass;
+ this.valueClass = valueClass;
+ }
+ @Override
+ public K convertKey(Record flinkRecord) {
+ if(flinkRecord.getNumFields() > 0) {
+ return convert(flinkRecord, 0, this.keyClass);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public V convertValue(Record flinkRecord) {
+ if(flinkRecord.getNumFields() > 1) {
+ return convert(flinkRecord, 1, this.valueClass);
+ } else {
+ return null;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private<T> T convert(Record flinkType, int pos, Class<T> hadoopType) {
+ if(hadoopType == LongWritable.class ) {
+ return (T) new LongWritable((flinkType.getField(pos, LongValue.class)).getValue());
+ }
+ if(hadoopType == org.apache.hadoop.io.Text.class) {
+ return (T) new Text((flinkType.getField(pos, StringValue.class)).getValue());
+ }
+ if(hadoopType == org.apache.hadoop.io.IntWritable.class) {
+ return (T) new IntWritable((flinkType.getField(pos, IntValue.class)).getValue());
+ }
+ if(hadoopType == org.apache.hadoop.io.FloatWritable.class) {
+ return (T) new FloatWritable((flinkType.getField(pos, FloatValue.class)).getValue());
+ }
+ if(hadoopType == org.apache.hadoop.io.DoubleWritable.class) {
+ return (T) new DoubleWritable((flinkType.getField(pos, DoubleValue.class)).getValue());
+ }
+ if(hadoopType == org.apache.hadoop.io.BooleanWritable.class) {
+ return (T) new BooleanWritable((flinkType.getField(pos, BooleanValue.class)).getValue());
+ }
+ if(hadoopType == org.apache.hadoop.io.ByteWritable.class) {
+ return (T) new ByteWritable((flinkType.getField(pos, ByteValue.class)).getValue());
+ }
+
+ throw new RuntimeException("Unable to convert Flink type ("+flinkType.getClass().getCanonicalName()+") to Hadoop.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultHadoopTypeConverter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultHadoopTypeConverter.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultHadoopTypeConverter.java
new file mode 100644
index 0000000..6ed670a
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultHadoopTypeConverter.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
+
+import org.apache.flink.types.BooleanValue;
+import org.apache.flink.types.ByteValue;
+import org.apache.flink.types.DoubleValue;
+import org.apache.flink.types.FloatValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.StringValue;
+import org.apache.flink.types.Value;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+
+
+/**
+ * Converter for the default hadoop writables.
+ * Key will be in field 0, Value in field 1 of a Record.
+ */
+public class DefaultHadoopTypeConverter<K, V> implements HadoopTypeConverter<K, V> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void convert(Record flinkRecord, K hadoopKey, V hadoopValue) {
+ flinkRecord.setField(0, convert(hadoopKey));
+ flinkRecord.setField(1, convert(hadoopValue));
+ }
+
+ protected Value convert(Object hadoopType) {
+ if(hadoopType instanceof org.apache.hadoop.io.LongWritable ) {
+ return new LongValue(((LongWritable)hadoopType).get());
+ }
+ if(hadoopType instanceof org.apache.hadoop.io.Text) {
+ return new StringValue(((Text)hadoopType).toString());
+ }
+ if(hadoopType instanceof org.apache.hadoop.io.IntWritable) {
+ return new IntValue(((IntWritable)hadoopType).get());
+ }
+ if(hadoopType instanceof org.apache.hadoop.io.FloatWritable) {
+ return new FloatValue(((FloatWritable)hadoopType).get());
+ }
+ if(hadoopType instanceof org.apache.hadoop.io.DoubleWritable) {
+ return new DoubleValue(((DoubleWritable)hadoopType).get());
+ }
+ if(hadoopType instanceof org.apache.hadoop.io.BooleanWritable) {
+ return new BooleanValue(((BooleanWritable)hadoopType).get());
+ }
+ if(hadoopType instanceof org.apache.hadoop.io.ByteWritable) {
+ return new ByteValue(((ByteWritable)hadoopType).get());
+ }
+ if (hadoopType instanceof NullWritable) {
+ return NullValue.getInstance();
+ }
+
+ throw new RuntimeException("Unable to convert Hadoop type ("+hadoopType.getClass().getCanonicalName()+") to a Flink data type.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/FlinkTypeConverter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/FlinkTypeConverter.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/FlinkTypeConverter.java
new file mode 100644
index 0000000..3c14a86
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/FlinkTypeConverter.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
+
+import java.io.Serializable;
+
+import org.apache.flink.types.Record;
+
+/**
+ * An interface describing a class that is able to
+ * convert Flink's Record into Hadoop types model.
+ *
+ * The converter must be Serializable.
+ *
+ * Flink provides a DefaultFlinkTypeConverter. Custom implementations should
+ * chain the type converters.
+ */
+public interface FlinkTypeConverter<K,V> extends Serializable {
+
+ /**
+ * Convert a Flink type to a Hadoop type.
+ */
+ public K convertKey(Record record);
+
+ public V convertValue(Record record);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopFileOutputCommitter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopFileOutputCommitter.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopFileOutputCommitter.java
new file mode 100644
index 0000000..ce4955c
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopFileOutputCommitter.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URI;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileOutputCommitter;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TaskAttemptID;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Hadoop 1.2.1 {@link org.apache.hadoop.mapred.FileOutputCommitter} takes {@link org.apache.hadoop.mapred.JobContext}
+ * as input parameter. However JobContext class is package private, and in Hadoop 2.2.0 it's public.
+ * This class takes {@link org.apache.hadoop.mapred.JobConf} as input instead of JobContext in order to setup and commit tasks.
+ */
+public class HadoopFileOutputCommitter extends FileOutputCommitter implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ static final String SUCCESSFUL_JOB_OUTPUT_DIR_MARKER =
+ "mapreduce.fileoutputcommitter.marksuccessfuljobs";
+
+ public void setupJob(JobConf conf) throws IOException {
+ Path outputPath = FileOutputFormat.getOutputPath(conf);
+ if (outputPath != null) {
+ Path tmpDir = new Path(outputPath, FileOutputCommitter.TEMP_DIR_NAME);
+ FileSystem fileSys = tmpDir.getFileSystem(conf);
+ if (!fileSys.mkdirs(tmpDir)) {
+ LOG.error("Mkdirs failed to create " + tmpDir.toString());
+ }
+ }
+ }
+
+ private static boolean getOutputDirMarking(JobConf conf) {
+ return conf.getBoolean(SUCCESSFUL_JOB_OUTPUT_DIR_MARKER, true);
+ }
+
+ private void markSuccessfulOutputDir(JobConf conf)
+ throws IOException {
+ Path outputPath = FileOutputFormat.getOutputPath(conf);
+ if (outputPath != null) {
+ FileSystem fileSys = outputPath.getFileSystem(conf);
+ // create a file in the folder to mark it
+ if (fileSys.exists(outputPath)) {
+ Path filePath = new Path(outputPath, SUCCEEDED_FILE_NAME);
+ fileSys.create(filePath).close();
+ }
+ }
+ }
+
+ private Path getFinalPath(Path jobOutputDir, Path taskOutput,
+ Path taskOutputPath) throws IOException {
+ URI taskOutputUri = taskOutput.toUri();
+ URI relativePath = taskOutputPath.toUri().relativize(taskOutputUri);
+ if (taskOutputUri == relativePath) {//taskOutputPath is not a parent of taskOutput
+ throw new IOException("Can not get the relative path: base = " +
+ taskOutputPath + " child = " + taskOutput);
+ }
+ if (relativePath.getPath().length() > 0) {
+ return new Path(jobOutputDir, relativePath.getPath());
+ } else {
+ return jobOutputDir;
+ }
+ }
+ private void moveTaskOutputs(JobConf conf, TaskAttemptID taskAttemptID,
+ FileSystem fs,
+ Path jobOutputDir,
+ Path taskOutput)
+ throws IOException {
+ if (fs.isFile(taskOutput)) {
+ Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput,
+ getTempTaskOutputPath(conf, taskAttemptID));
+ if (!fs.rename(taskOutput, finalOutputPath)) {
+ if (!fs.delete(finalOutputPath, true)) {
+ throw new IOException("Failed to delete earlier output of task: " +
+ taskAttemptID);
+ }
+ if (!fs.rename(taskOutput, finalOutputPath)) {
+ throw new IOException("Failed to save output of task: " +
+ taskAttemptID);
+ }
+ }
+ LOG.debug("Moved " + taskOutput + " to " + finalOutputPath);
+ } else if(fs.getFileStatus(taskOutput).isDir()) {
+ FileStatus[] paths = fs.listStatus(taskOutput);
+ Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput,
+ getTempTaskOutputPath(conf, taskAttemptID));
+ fs.mkdirs(finalOutputPath);
+ if (paths != null) {
+ for (FileStatus path : paths) {
+ moveTaskOutputs(conf,taskAttemptID, fs, jobOutputDir, path.getPath());
+ }
+ }
+ }
+ }
+
+ public void commitTask(JobConf conf, TaskAttemptID taskAttemptID)
+ throws IOException {
+ Path taskOutputPath = getTempTaskOutputPath(conf, taskAttemptID);
+ if (taskOutputPath != null) {
+ FileSystem fs = taskOutputPath.getFileSystem(conf);
+ if (fs.exists(taskOutputPath)) {
+ Path jobOutputPath = taskOutputPath.getParent().getParent();
+ // Move the task outputs to their final place
+ moveTaskOutputs(conf,taskAttemptID, fs, jobOutputPath, taskOutputPath);
+ // Delete the temporary task-specific output directory
+ if (!fs.delete(taskOutputPath, true)) {
+ LOG.info("Failed to delete the temporary output" +
+ " directory of task: " + taskAttemptID + " - " + taskOutputPath);
+ }
+ LOG.info("Saved output of task '" + taskAttemptID + "' to " +
+ jobOutputPath);
+ }
+ }
+ }
+ public boolean needsTaskCommit(JobConf conf, TaskAttemptID taskAttemptID)
+ throws IOException {
+ try {
+ Path taskOutputPath = getTempTaskOutputPath(conf, taskAttemptID);
+ if (taskOutputPath != null) {
+ // Get the file-system for the task output directory
+ FileSystem fs = taskOutputPath.getFileSystem(conf);
+ // since task output path is created on demand,
+ // if it exists, task needs a commit
+ if (fs.exists(taskOutputPath)) {
+ return true;
+ }
+ }
+ } catch (IOException ioe) {
+ throw ioe;
+ }
+ return false;
+ }
+
+ public Path getTempTaskOutputPath(JobConf conf, TaskAttemptID taskAttemptID) {
+ Path outputPath = FileOutputFormat.getOutputPath(conf);
+ if (outputPath != null) {
+ Path p = new Path(outputPath,
+ (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR +
+ "_" + taskAttemptID.toString()));
+ try {
+ FileSystem fs = p.getFileSystem(conf);
+ return p.makeQualified(fs);
+ } catch (IOException ie) {
+ LOG.warn(StringUtils.stringifyException(ie));
+ return p;
+ }
+ }
+ return null;
+ }
+ public void cleanupJob(JobConf conf) throws IOException {
+ // do the clean up of temporary directory
+ Path outputPath = FileOutputFormat.getOutputPath(conf);
+ if (outputPath != null) {
+ Path tmpDir = new Path(outputPath, FileOutputCommitter.TEMP_DIR_NAME);
+ FileSystem fileSys = tmpDir.getFileSystem(conf);
+ if (fileSys.exists(tmpDir)) {
+ fileSys.delete(tmpDir, true);
+ }
+ } else {
+ LOG.warn("Output path is null in cleanup");
+ }
+ }
+
+ public void commitJob(JobConf conf) throws IOException {
+ cleanupJob(conf);
+ if (getOutputDirMarking(conf)) {
+ markSuccessfulOutputDir(conf);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopTypeConverter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopTypeConverter.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopTypeConverter.java
new file mode 100644
index 0000000..6bbf077
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopTypeConverter.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
+
+import java.io.Serializable;
+
+import org.apache.flink.types.Record;
+
+
+/**
+ * An interface describing a class that is able to
+ * convert Hadoop types into Flink's Record model.
+ *
+ * The converter must be Serializable.
+ *
+ * Flink provides a DefaultHadoopTypeConverter. Custom implementations should
+ * chain the type converters.
+ */
+public interface HadoopTypeConverter<K, V> extends Serializable {
+
+ /**
+ * Convert a Hadoop type to a Flink type.
+ */
+ public void convert(Record record, K hadoopKey, V hadoopValue);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableComparableWrapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableComparableWrapper.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableComparableWrapper.java
new file mode 100644
index 0000000..0519ac9
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableComparableWrapper.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
+
+import org.apache.flink.types.Key;
+import org.apache.hadoop.io.WritableComparable;
+
+public class WritableComparableWrapper<T extends WritableComparable<T>> extends WritableWrapper<T> implements Key<WritableComparableWrapper<T>> {
+ private static final long serialVersionUID = 1L;
+
+ public WritableComparableWrapper() {
+ super();
+ }
+
+ public WritableComparableWrapper(T toWrap) {
+ super(toWrap);
+ }
+
+ @Override
+ public int compareTo(WritableComparableWrapper<T> o) {
+ return super.value().compareTo(o.value());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapper.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapper.java
new file mode 100644
index 0000000..6369bb7
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapper.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
+
+import java.io.IOException;
+
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.types.Value;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.hadoop.io.Writable;
+
+public class WritableWrapper<T extends Writable> implements Value {
+ private static final long serialVersionUID = 2L;
+
+ private T wrapped;
+ private String wrappedType;
+ private ClassLoader cl;
+
+ public WritableWrapper() {
+ }
+
+ public WritableWrapper(T toWrap) {
+ wrapped = toWrap;
+ wrappedType = toWrap.getClass().getCanonicalName();
+ }
+
+ public T value() {
+ return wrapped;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ out.writeUTF(wrappedType);
+ wrapped.write(out);
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+ if(cl == null) {
+ cl = Thread.currentThread().getContextClassLoader();
+ }
+ wrappedType = in.readUTF();
+ try {
+ @SuppressWarnings("unchecked")
+ Class<T> wrClass = (Class<T>) Class.forName(wrappedType, true, cl).asSubclass(Writable.class);
+ wrapped = InstantiationUtil.instantiate(wrClass, Writable.class);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Error creating the WritableWrapper", e);
+ }
+ wrapped.readFields(in);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapperConverter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapperConverter.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapperConverter.java
new file mode 100644
index 0000000..6bb13d1
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/WritableWrapperConverter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
+
+import org.apache.flink.types.Record;
+import org.apache.flink.types.Value;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+@SuppressWarnings("rawtypes")
+public class WritableWrapperConverter<K extends WritableComparable, V extends Writable> implements HadoopTypeConverter<K,V> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void convert(Record flinkRecord, K hadoopKey, V hadoopValue) {
+ flinkRecord.setField(0, convertKey(hadoopKey));
+ flinkRecord.setField(1, convertValue(hadoopValue));
+ }
+
+ @SuppressWarnings("unchecked")
+ private final Value convertKey(K in) {
+ return new WritableComparableWrapper(in);
+ }
+
+ private final Value convertValue(V in) {
+ return new WritableWrapper<V>(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java
new file mode 100644
index 0000000..b167080
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapred.record.example;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.Program;
+import org.apache.flink.api.common.ProgramDescription;
+import org.apache.flink.api.java.record.functions.MapFunction;
+import org.apache.flink.api.java.record.functions.ReduceFunction;
+import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFields;
+import org.apache.flink.api.java.record.io.CsvOutputFormat;
+import org.apache.flink.api.java.record.operators.FileDataSink;
+import org.apache.flink.api.java.record.operators.MapOperator;
+import org.apache.flink.api.java.record.operators.ReduceOperator;
+import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
+import org.apache.flink.client.LocalExecutor;
+import org.apache.flink.hadoopcompatibility.mapred.record.HadoopDataSource;
+import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.WritableWrapperConverter;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.StringValue;
+import org.apache.flink.util.Collector;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+/**
+ * Implements a word count which takes the input file and counts the number of
+ * the occurrences of each word in the file.
+ *
+ * <br /><br />
+ *
+ * <b>Note</b>: This example uses the out-dated Record API.
+ * It is recommended to use the new Java API.
+ *
+ * @see org.apache.flink.hadoopcompatibility.mapred.example.HadoopMapredCompatWordCount
+ */
+public class WordCount implements Program, ProgramDescription {
+
+ private static final long serialVersionUID = 1L;
+
+
+ /**
+ * Converts a Record containing one string in to multiple string/integer pairs.
+ * The string is tokenized by whitespaces. For each token a new record is emitted,
+ * where the token is the first field and an Integer(1) is the second field.
+ */
+ public static class TokenizeLine extends MapFunction implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void map(Record record, Collector<Record> collector) {
+ // get the first field (as type StringValue) from the record
+ String line = record.getField(1, StringValue.class).getValue();
+ // normalize the line
+ line = line.replaceAll("\\W+", " ").toLowerCase();
+
+ // tokenize the line
+ StringTokenizer tokenizer = new StringTokenizer(line);
+ while (tokenizer.hasMoreTokens()) {
+ String word = tokenizer.nextToken();
+
+ // we emit a (word, 1) pair
+ collector.collect(new Record(new StringValue(word), new IntValue(1)));
+ }
+ }
+ }
+
+ /**
+ * Sums up the counts for a certain given key. The counts are assumed to be at position <code>1</code>
+ * in the record. The other fields are not modified.
+ */
+ @Combinable
+ @ConstantFields(0)
+ public static class CountWords extends ReduceFunction implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
+ Record element = null;
+ int sum = 0;
+
+ while (records.hasNext()) {
+ element = records.next();
+ int cnt = element.getField(1, IntValue.class).getValue();
+ sum += cnt;
+ }
+
+ element.setField(1, new IntValue(sum));
+ out.collect(element);
+ }
+
+ @Override
+ public void combine(Iterator<Record> records, Collector<Record> out) throws Exception {
+ // the logic is the same as in the reduce function, so simply call the reduce method
+ reduce(records, out);
+ }
+ }
+
+
+ @SuppressWarnings({ "rawtypes", "unchecked", "unused" })
+ @Override
+ public Plan getPlan(String... args) {
+ // parse job parameters
+ int numSubTasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
+ String dataInput = (args.length > 1 ? args[1] : "");
+ String output = (args.length > 2 ? args[2] : "");
+
+
+ HadoopDataSource source = new HadoopDataSource(new TextInputFormat(), new JobConf(), "Input Lines");
+ TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
+
+ // Example with Wrapper Converter
+ HadoopDataSource<LongWritable,Text> sourceHadoopType = new HadoopDataSource<LongWritable, Text>(
+ new TextInputFormat(), new JobConf(), "Input Lines", new WritableWrapperConverter<LongWritable, Text>());
+ TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
+
+ MapOperator mapper = MapOperator.builder(new TokenizeLine())
+ .input(source)
+ .name("Tokenize Lines")
+ .build();
+ ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
+ .input(mapper)
+ .name("Count Words")
+ .build();
+ FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, reducer, "Word Counts");
+ CsvOutputFormat.configureRecordFormat(out)
+ .recordDelimiter('\n')
+ .fieldDelimiter(' ')
+ .field(StringValue.class, 0)
+ .field(IntValue.class, 1);
+
+ Plan plan = new Plan(out, "WordCount Example");
+ plan.setDefaultParallelism(numSubTasks);
+ return plan;
+ }
+
+
+ @Override
+ public String getDescription() {
+ return "Parameters: [numSubStasks] [input] [output]";
+ }
+
+
+ public static void main(String[] args) throws Exception {
+ WordCount wc = new WordCount();
+
+ if (args.length < 3) {
+ System.err.println(wc.getDescription());
+ System.exit(1);
+ }
+
+ Plan plan = wc.getPlan(args);
+
+ // This will execute the word-count embedded in a local context. replace this line by the commented
+ // succeeding line to send the job to a local installation or to a cluster for execution
+ LocalExecutor.execute(plan);
+// PlanExecutor ex = new RemoteExecutor("localhost", 6123, "target/pact-examples-0.4-SNAPSHOT-WordCount.jar");
+// ex.executePlan(plan);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java
new file mode 100644
index 0000000..a838215
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapred.record.example;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.Program;
+import org.apache.flink.api.common.ProgramDescription;
+import org.apache.flink.api.java.record.functions.MapFunction;
+import org.apache.flink.api.java.record.functions.ReduceFunction;
+import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFields;
+import org.apache.flink.api.java.record.operators.MapOperator;
+import org.apache.flink.api.java.record.operators.ReduceOperator;
+import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
+import org.apache.flink.client.LocalExecutor;
+import org.apache.flink.hadoopcompatibility.mapred.record.HadoopDataSink;
+import org.apache.flink.hadoopcompatibility.mapred.record.HadoopDataSource;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.StringValue;
+import org.apache.flink.util.Collector;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+
+/**
+ * Implements a word count which takes the input file and counts the number of
+ * the occurrences of each word in the file.
+ *
+ * <br /><br />
+ *
+ * <b>Note</b>: This example uses the out dated Record API.
+ * It is recommended to use the new Java API.
+ *
+ * @see WordCount
+ */
+@SuppressWarnings("serial")
+public class WordCountWithOutputFormat implements Program, ProgramDescription {
+
+ /**
+ * Converts a Record containing one string in to multiple string/integer pairs.
+ * The string is tokenized by whitespaces. For each token a new record is emitted,
+ * where the token is the first field and an Integer(1) is the second field.
+ */
+ public static class TokenizeLine extends MapFunction implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void map(Record record, Collector<Record> collector) {
+ // get the first field (as type StringValue) from the record
+ String line = record.getField(1, StringValue.class).getValue();
+ // normalize the line
+ line = line.replaceAll("\\W+", " ").toLowerCase();
+
+ // tokenize the line
+ StringTokenizer tokenizer = new StringTokenizer(line);
+ while (tokenizer.hasMoreTokens()) {
+ String word = tokenizer.nextToken();
+
+ // we emit a (word, 1) pair
+ collector.collect(new Record(new StringValue(word), new IntValue(1)));
+ }
+ }
+ }
+
+ /**
+ * Sums up the counts for a certain given key. The counts are assumed to be at position <code>1</code>
+ * in the record. The other fields are not modified.
+ */
+ @Combinable
+ @ConstantFields(0)
+ public static class CountWords extends ReduceFunction implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
+ Record element = null;
+ int sum = 0;
+
+ while (records.hasNext()) {
+ element = records.next();
+ int cnt = element.getField(1, IntValue.class).getValue();
+ sum += cnt;
+ }
+
+ element.setField(1, new IntValue(sum));
+ out.collect(element);
+ }
+
+ @Override
+ public void combine(Iterator<Record> records, Collector<Record> out) throws Exception {
+ // the logic is the same as in the reduce function, so simply call the reduce method
+ reduce(records, out);
+ }
+ }
+
+
+ @Override
+ public Plan getPlan(String... args) {
+ // parse job parameters
+ int numSubTasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
+ String dataInput = (args.length > 1 ? args[1] : "");
+ String output = (args.length > 2 ? args[2] : "");
+
+ HadoopDataSource<LongWritable, Text> source = new HadoopDataSource<LongWritable, Text>(
+ new TextInputFormat(), new JobConf(), "Input Lines");
+ TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
+
+
+ MapOperator mapper = MapOperator.builder(new TokenizeLine())
+ .input(source)
+ .name("Tokenize Lines")
+ .build();
+ ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
+ .input(mapper)
+ .name("Count Words")
+ .build();
+ HadoopDataSink<Text, IntWritable> out = new HadoopDataSink<Text, IntWritable>(new TextOutputFormat<Text, IntWritable>(),new JobConf(), "Hadoop TextOutputFormat", reducer, Text.class, IntWritable.class);
+ TextOutputFormat.setOutputPath(out.getJobConf(), new Path(output));
+
+ Plan plan = new Plan(out, "Hadoop OutputFormat Example");
+ plan.setDefaultParallelism(numSubTasks);
+ return plan;
+ }
+
+
+ @Override
+ public String getDescription() {
+ return "Parameters: [numSubStasks] [input] [output]";
+ }
+
+
+ public static void main(String[] args) throws Exception {
+ WordCountWithOutputFormat wc = new WordCountWithOutputFormat();
+
+ if (args.length < 3) {
+ System.err.println(wc.getDescription());
+ System.exit(1);
+ }
+
+ Plan plan = wc.getPlan(args);
+
+ // This will execute the word-count embedded in a local context. replace this line by the commented
+ // succeeding line to send the job to a local installation or to a cluster for execution
+ LocalExecutor.execute(plan);
+// PlanExecutor ex = new RemoteExecutor("localhost", 6123, "target/pact-examples-0.4-SNAPSHOT-WordCount.jar");
+// ex.executePlan(plan);
+ }
+}
[29/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/util/TwitterStreamData.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/util/TwitterStreamData.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/util/TwitterStreamData.java
deleted file mode 100644
index 0e7976c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/util/TwitterStreamData.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.twitter.util;
-
-public class TwitterStreamData {
- public static final String[] TEXTS = new String[] {
- "{\"delete\":{\"status\":{\"id\":448896216110080001,\"id_str\":\"448896216110080001\",\"user_id\":1586900228,\"user_id_str\":\"1586900228\"},\"timestamp_ms\":\"1412178007914\"}}",
- "{\"created_at\":\"Wed Oct 01 15:40:07 +0000 2014\",\"id\":517338176713523201,\"id_str\":\"517338176713523201\",\"text\":\"RT @IniAnwarHadi: If you're naive enough to think that we would be better off if the opposition won the last elections, then good for you.\",\"source\":\"\\u003ca href=\\\"https:\\/\\/twitter.com\\/download\\/android\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Android\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":1402119228,\"id_str\":\"1402119228\",\"name\":\"\\u6cd5 \\u8482 \\u54c8\",\"screen_name\":\"sesepiSepi\",\"location\":\"Malaysia || Edinburgh\",\"url\":null,\"description\":\"To whom are you enslaving yourself?\",\"protected\":false,\"verified\":false,\"followers_count\":255,\"friends_count\":197,\"listed_count\":2,\"favourites_count\":4064,\"statuses_count\":30382,\"created_at\":\
"Sat May 04 12:01:31 +0000 2013\",\"utc_offset\":28800,\"time_zone\":\"Beijing\",\"geo_enabled\":true,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"FF6699\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/436368673033306112\\/gR_FAlFW.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/436368673033306112\\/gR_FAlFW.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"B40B43\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/510951209868337152\\/6eNXVNb__normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/510951209868337152\\/6eNXVNb__normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banner
s\\/1402119228\\/1409317654\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:17:59 +0000 2014\",\"id\":517332604706508800,\"id_str\":\"517332604706508800\",\"text\":\"If you're naive enough to think that we would be better off if the opposition won the last elections, then good for you.\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":234395084,\"id_str\":\"234395084\",\"name\":\"Anwar Hadi\",\"screen_name\":\"IniAnwarHadi\",\"location\":\"Pulau Pinang\",\"url\":\"http:\\/\\/www.anakazman.b
logspot.com\",\"description\":\"Sometimes a blogger, sometimes a vlogger, sometimes a noob teacher, always a Muslim. Managed by Gushcloud. @DeludeOfficial\",\"protected\":false,\"verified\":false,\"followers_count\":102098,\"friends_count\":289,\"listed_count\":210,\"favourites_count\":186,\"statuses_count\":45071,\"created_at\":\"Wed Jan 05 15:15:07 +0000 2011\",\"utc_offset\":28800,\"time_zone\":\"Kuala Lumpur\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_u
rl\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/456689283240189952\\/ldMM1VUP_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/456689283240189952\\/ldMM1VUP_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/234395084\\/1349837992\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":75,\"favorite_count\":42,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"en\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"IniAnwarHadi\",\"name\":\"Anwar Hadi\",\"id\":234395084,\"id_str\":\"234395084\",\"indices\":[3,16]}],\"s
ymbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_ms\":\"1412178007807\"}",
- "{\"delete\":{\"status\":{\"id\":516592555278872576,\"id_str\":\"516592555278872576\",\"user_id\":2835411134,\"user_id_str\":\"2835411134\"},\"timestamp_ms\":\"1412178007920\"}}",
- "{\"created_at\":\"Wed Oct 01 15:40:07 +0000 2014\",\"id\":517338176709345280,\"id_str\":\"517338176709345280\",\"text\":\"RT @takaaaki0410: \\u697d\\u3057\\u305d\\u3046\\u306b\\u9699\\u9593\\u30c1\\u30a7\\u30c3\\u30af\\u3092\\u3059\\u308b\\u3061\\u3073\\u3046\\u305d\\u30ba\\u3002\\u304b\\u308f\\u3046\\u305d\\u3089\\u3057\\u304f\\u306a\\u3063\\u3066\\u304d\\u307e\\u3057\\u305f\\u3002#\\u30b5\\u30f3\\u30b7\\u30e3\\u30a4\\u30f3\\u6c34\\u65cf\\u9928 http:\\/\\/t.co\\/McsvrHPAr0\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\\" rel=\\\"nofollow\\\"\\u003eTwitter Web Client\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":118319345,\"id_str\":\"118319345\",\"name\":\"leonardo\",\"screen_name\":\"masumind\",\"location\":\"Japan\",\"url\":null,\"description\":\"Working for strategic designing. Once lived in U.S
.A., U.S.S.R. and Russia. Now in Japan. Part-time university lecturer \\u500b\\u4eba\\u306e\\u7acb\\u5834\\u3067\\u3064\\u3076\\u3084\\u3044\\u3066\\u3044\\u307e\\u3059\\u3002\",\"protected\":false,\"verified\":false,\"followers_count\":1604,\"friends_count\":597,\"listed_count\":131,\"favourites_count\":600,\"statuses_count\":27957,\"created_at\":\"Sun Feb 28 08:33:20 +0000 2010\",\"utc_offset\":32400,\"time_zone\":\"Tokyo\",\"geo_enaboutputPathled\":false,\"lang\":\"ja\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"ACDED6\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/282110252\\/IMG_4869.JPG\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/282110252\\/IMG_4869.JPG\",\"profile_background_tile\":true,\"profile_link_color\":\"038543\",\"profile_sidebar_border_color\":\"EEEEEE\",\"profile_sidebar_fill_color\":\"F6F6F6\",\"profile_text_color\":\"33333
3\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/1423205970\\/2006_8___________103_normal.jpg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/1423205970\\/2006_8___________103_normal.jpg\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 14:17:39 +0000 2014\",\"id\":517317423549018112,\"id_str\":\"517317423549018112\",\"text\":\"\\u697d\\u3057\\u305d\\u3046\\u306b\\u9699\\u9593\\u30c1\\u30a7\\u30c3\\u30af\\u3092\\u3059\\u308b\\u3061\\u3073\\u3046\\u305d\\u30ba\\u3002\\u304b\\u308f\\u3046\\u305d\\u3089\\u3057\\u304f\\u306a\\u3063\\u3066\\u304d\\u307e\\u3057\\u305f\\u3002#\\u30b5\\u30f3\\u30b7\\u30e3\\u30a4\\u30f3\\u6c34\\u65cf\\u9928 http:\\/\\/t.co\\/McsvrHPAr0\",\"source\":\"\\u003ca href=\\\"ht
tp:\\/\\/twitter.com\\\" rel=\\\"nofollow\\\"\\u003eTwitter Web Client\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":265723969,\"id_str\":\"265723969\",\"name\":\"Takaaki Sawada\",\"screen_name\":\"takaaaki0410\",\"location\":\"\",\"url\":null,\"description\":\"\\u304a\\u3044\\u3057\\u3044\\u3082\\u306e\\u3068\\u30ab\\u30ef\\u30a6\\u30bd\\u304c\\u597d\\u304d\\u3067\\u3059\\u3002\",\"protected\":false,\"verified\":false,\"followers_count\":352,\"friends_count\":197,\"listed_count\":13,\"favourites_count\":366,\"statuses_count\":6634,\"created_at\":\"Mon Mar 14 01:54:33 +0000 2011\",\"utc_offset\":32400,\"time_zone\":\"Tokyo\",\"geo_enabled\":false,\"lang\":\"ja\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"000000\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com
\\/profile_background_images\\/749259666\\/9ebe30d2ac8d95d6266181ab0f61f90c.png\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/749259666\\/9ebe30d2ac8d95d6266181ab0f61f90c.png\",\"profile_background_tile\":true,\"profile_link_color\":\"599BFF\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"585959\",\"profile_text_color\":\"FF82E8\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/415878656800460801\\/OSLSTJcC_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/415878656800460801\\/OSLSTJcC_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/265723969\\/1398242024\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_coun
t\":69,\"favorite_count\":79,\"entities\":{\"hashtags\":[{\"text\":\"\\u30b5\\u30f3\\u30b7\\u30e3\\u30a4\\u30f3\\u6c34\\u65cf\\u9928\",\"indices\":[35,45]}],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[],\"media\":[{\"id\":517317416271880192,\"id_str\":\"517317416271880192\",\"indices\":[46,68],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}}}]},\"extended_entities\":{\"media\":[{\"id\":517317416271880192,\"id_str\":\"517317416271880192\",\"indi
ces\":[46,68],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}}},{\"id\":517317413965004800,\"id_str\":\"517317413965004800\",\"indices\":[46,68],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcneCAAA-HPL.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcneCAAA-HPL.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/51731742
3549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}}},{\"id\":517317407241543681,\"id_str\":\"517317407241543681\",\"indices\":[46,68],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcObCIAEm57j.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcObCIAEm57j.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}}}]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":fals
e,\"filter_level\":\"low\",\"lang\":\"ja\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"\\u30b5\\u30f3\\u30b7\\u30e3\\u30a4\\u30f3\\u6c34\\u65cf\\u9928\",\"indices\":[53,63]}],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"takaaaki0410\",\"name\":\"Takaaki Sawada\",\"id\":265723969,\"id_str\":\"265723969\",\"indices\":[3,16]}],\"symbols\":[],\"media\":[{\"id\":517317416271880192,\"id_str\":\"517317416271880192\",\"indices\":[64,86],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":15
0,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}},\"source_status_id\":517317423549018112,\"source_status_id_str\":\"517317423549018112\"}]},\"extended_entities\":{\"media\":[{\"id\":517317416271880192,\"id_str\":\"517317416271880192\",\"indices\":[64,86],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcwECIAA7e5T.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}},\"source_status_id\":517317423549018112,\"source_status_id_str\":\"517317423549018112\"},{\"id\":517317413965004800,\"id_str\":\"51731
7413965004800\",\"indices\":[64,86],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcneCAAA-HPL.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcneCAAA-HPL.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\":\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}},\"source_status_id\":517317423549018112,\"source_status_id_str\":\"517317423549018112\"},{\"id\":517317407241543681,\"id_str\":\"517317407241543681\",\"indices\":[64,86],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3hcObCIAEm57j.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3hcObCIAEm57j.png\",\"url\":\"http:\\/\\/t.co\\/McsvrHPAr0\",\"display_url\"
:\"pic.twitter.com\\/McsvrHPAr0\",\"expanded_url\":\"http:\\/\\/twitter.com\\/takaaaki0410\\/status\\/517317423549018112\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":600,\"h\":400,\"resize\":\"fit\"},\"large\":{\"w\":1024,\"h\":683,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":226,\"resize\":\"fit\"}},\"source_status_id\":517317423549018112,\"source_status_id_str\":\"517317423549018112\"}]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"ja\",\"timestamp_ms\":\"1412178007863\"}",
- "{\"delete\":{\"status\":{\"id\":517192344945192960,\"id_str\":\"517192344945192960\",\"user_id\":1134175530,\"user_id_str\":\"1134175530\"},\"timestamp_ms\":\"1412178007972\"}}",
- "{\"delete\":{\"status\":{\"id\":431653696582078464,\"id_str\":\"431653696582078464\",\"user_id\":1028327856,\"user_id_str\":\"1028327856\"},\"timestamp_ms\":\"1412178007990\"}}",
- "{\"delete\":{\"status\":{\"id\":517337912464408576,\"id_str\":\"517337912464408576\",\"user_id\":607266683,\"user_id_str\":\"607266683\"},\"timestamp_ms\":\"1412178008106\"}}",
- "{\"delete\":{\"status\":{\"id\":517191707427758080,\"id_str\":\"517191707427758080\",\"user_id\":268787345,\"user_id_str\":\"268787345\"},\"timestamp_ms\":\"1412178008127\"}}",
- "{\"delete\":{\"status\":{\"id\":382158053065256961,\"id_str\":\"382158053065256961\",\"user_id\":1303521211,\"user_id_str\":\"1303521211\"},\"timestamp_ms\":\"1412178008294\"}}",
- "{\"delete\":{\"status\":{\"id\":421056787530280960,\"id_str\":\"421056787530280960\",\"user_id\":63286510,\"user_id_str\":\"63286510\"},\"timestamp_ms\":\"1412178008377\"}}",
- "{\"delete\":{\"status\":{\"id\":450659899941584898,\"id_str\":\"450659899941584898\",\"user_id\":257093107,\"user_id_str\":\"257093107\"},\"timestamp_ms\":\"1412178008454\"}}",
- "{\"delete\":{\"status\":{\"id\":450515712348790784,\"id_str\":\"450515712348790784\",\"user_id\":257093107,\"user_id_str\":\"257093107\"},\"timestamp_ms\":\"1412178008458\"}}",
- "{\"delete\":{\"status\":{\"id\":517190671422087168,\"id_str\":\"517190671422087168\",\"user_id\":1147896566,\"user_id_str\":\"1147896566\"},\"timestamp_ms\":\"1412178008435\"}}",
- "{\"delete\":{\"status\":{\"id\":324613502436257793,\"id_str\":\"324613502436257793\",\"user_id\":634892640,\"user_id_str\":\"634892640\"},\"timestamp_ms\":\"1412178008499\"}}",
- "{\"delete\":{\"status\":{\"id\":497003314308186112,\"id_str\":\"497003314308186112\",\"user_id\":451705771,\"user_id_str\":\"451705771\"},\"timestamp_ms\":\"1412178008523\"}}",
- "{\"delete\":{\"status\":{\"id\":516590386823696385,\"id_str\":\"516590386823696385\",\"user_id\":1556958698,\"user_id_str\":\"1556958698\"},\"timestamp_ms\":\"1412178008623\"}}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180895662080,\"id_str\":\"517338180895662080\",\"text\":\"yolunu bulmaktan umudunu kesme, \\u00e7\\u00fcnk\\u00fc umut,tur insana yol g\\u00f6steren\\n G\\u00f6klerdeY\\u0131ld\\u0131zKalplerdeAy T\\u00fcrkiyedirGalatasaray\",\"source\":\"\\u003ca href=\\\"https:\\/\\/twitter.com\\/\\\" rel=\\\"nofollow\\\"\\u003eMobile Webs\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":712720935,\"id_str\":\"712720935\",\"name\":\"Bensuu\",\"screen_name\":\"bnsukara\",\"location\":\"YASEM\\u0130N HER\\u015eEY\\u0130M\",\"url\":\"https:\\/\\/www.facebook.com\\/Webmaster99\",\"description\":\"EMRE POLAT TAK\\u0130P EDEN TAK\\u0130PC\\u0130 KAZANIYOR ----@emrepolatt1\",\"protected\":false,\"verified\":false,\"followers_count\":422,\"friends_count\":327,\"listed_coun
t\":0,\"favourites_count\":4960,\"statuses_count\":3561,\"created_at\":\"Sun Oct 13 14:14:03 +0000 2013\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"tr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/378800000113509334\\/492598cb8f47b2a29fe392d7e9eabb44.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/378800000113509334\\/492598cb8f47b2a29fe392d7e9eabb44.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/516711945240838144\\/1MXxVPoJ_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\
\/516711945240838144\\/1MXxVPoJ_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/712720935\\/1402706181\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"tr\",\"timestamp_ms\":\"1412178008664\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180887261184,\"id_str\":\"517338180887261184\",\"text\":\"#Bug\\u00fcnG\\u00fcnlerdenAvrupan\\u0131nFethiGALATASARAY\\u0131m\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":295319165,\"id_str\":\"295319165\",\"name\":\"Do\\u011fa Marangoz 1905\",\"screen_name\":\"doa1905\",\"location\":\"\",\"url\":null,\"description\":\"Cehennem donana kadar #GALATASARAY N.K.F.V.A.S -Sar\\u0131yer Do\\u011fa Koleji #HamitAltintop #FelipeMelo #BurakY\\u0131lmaz #OlcanAd\\u0131n 30.03.2012 T.D OMERNAZ \\u0130stanbul\\/Sar\\u0131yer\",\"protected\":false,\"verified\":false,\"followers_count\":578,\"friends_count\":272,\"listed_count\":0,\"favou
rites_count\":7928,\"statuses_count\":12005,\"created_at\":\"Sun May 08 19:51:11 +0000 2011\",\"utc_offset\":-18000,\"time_zone\":\"Quito\",\"geo_enabled\":false,\"lang\":\"tr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/814478464\\/5ea45052ee5cdc7d424ede463bd88795.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/814478464\\/5ea45052ee5cdc7d424ede463bd88795.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/513713202191163392\\/dKiNGqS-_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/513713202191163392\\/
dKiNGqS-_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/295319165\\/1397810158\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"Bug\\u00fcnG\\u00fcnlerdenAvrupan\\u0131nFethiGALATASARAY\\u0131m\",\"indices\":[0,42]}],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"und\",\"timestamp_ms\":\"1412178008666\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180883058688,\"id_str\":\"517338180883058688\",\"text\":\"Un par de zapatos, dos estilos http:\\/\\/t.co\\/SOGGZDmTrk\",\"source\":\"\\u003ca href=\\\"http:\\/\\/www.wonderland.fm\\/\\\" rel=\\\"nofollow\\\"\\u003ewonderland.fm App\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":1135415474,\"id_str\":\"1135415474\",\"name\":\"CalzandoZapatos\",\"screen_name\":\"CalzandoZapatos\",\"location\":\"\",\"url\":null,\"description\":\"Sobre todo tipo de zapatos y calzado. Encontraras lo m\\u00e1s nuevo en noticias, y todo tipo de ofertas\",\"protected\":false,\"verified\":false,\"followers_count\":10181,\"friends_count\":10975,\"listed_count\":15,\"favourites_count\":0,\"statuses_count\":5434,\"created_at\":\"Wed Jan 30 22:51:24 +0000 2013\",\"utc_offset\":
7200,\"time_zone\":\"Amsterdam\",\"geo_enabled\":false,\"lang\":\"es\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"D1B6D1\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme17\\/bg.gif\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme17\\/bg.gif\",\"profile_background_tile\":false,\"profile_link_color\":\"CC3366\",\"profile_sidebar_border_color\":\"000000\",\"profile_sidebar_fill_color\":\"E6F6F9\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":false,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/3197686515\\/08837c6a23605352e2bd73c7890cb54d_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/3197686515\\/08837c6a23605352e2bd73c7890cb54d_normal.jpeg\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\
":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[{\"url\":\"http:\\/\\/t.co\\/SOGGZDmTrk\",\"expanded_url\":\"http:\\/\\/bit.ly\\/1rGjI18\",\"display_url\":\"bit.ly\\/1rGjI18\",\"indices\":[31,53]}],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"es\",\"timestamp_ms\":\"1412178008663\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180887257088,\"id_str\":\"517338180887257088\",\"text\":\"\\u0412\\u043c\\u0435\\u0441\\u0442\\u043e \\u044d\\u0442\\u043e\\u0433\\u043e \\u0440\\u043e\\u0434\\u0438\\u0442\\u0435\\u043b\\u0438 \\u0434\\u043e\\u043b\\u0436\\u043d\\u044b \\u043f\\u0435\\u0440\\u0435\\u043d\\u0430\\u043f\\u0440\\u0430\\u0432\\u0438\\u0442\\u044c \\u0434\\u0435\\u0442\\u0435\\u0439.\",\"source\":\"\\u003ca href=\\\"http:\\/\\/barlear.ru\\/\\\" rel=\\\"nofollow\\\"\\u003eSocialNetworkMarketingApp\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":350347678,\"id_str\":\"350347678\",\"name\":\"murat g\\u00fcndo\\u011fdu\",\"screen_name\":\"mrt_mrt_\",\"location\":\"\",\"url\":null,\"description\":null,\"protected\":false,\"verified\":false,\"followers_count\":369,\"friends_c
ount\":387,\"listed_count\":1,\"favourites_count\":0,\"statuses_count\":1691,\"created_at\":\"Sun Aug 07 16:36:54 +0000 2011\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"tr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/1482992087\\/58537_148894648483546_100000890344526_246985_5158298_n_normal.jpg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/1482992087\\/58537_148894648483546_1000008903
44526_246985_5158298_n_normal.jpg\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"ru\",\"timestamp_ms\":\"1412178008664\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180878860288,\"id_str\":\"517338180878860288\",\"text\":\"RT @jennybethm: Illegal Immigration: 70 Percent of Released Families Never Report Back To Federal Agents http:\\/\\/t.co\\/89DNq2IUFK #teaparty\",\"source\":\"\\u003ca href=\\\"https:\\/\\/roundteam.co\\\" rel=\\\"nofollow\\\"\\u003eRoundTeam\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":596541160,\"id_str\":\"596541160\",\"name\":\"Choose Freedom\",\"screen_name\":\"choose__freedom\",\"location\":\"Where they can't find me.\",\"url\":null,\"description\":\"I believe in #freedom, limited government, a strong national defense, secure borders + personal responsibility. #tcot #tlot #PJNET #NRA #2A #TeaParty #TGDN\",\"protected\":false,\"verified\":false,\"followers_count\":1552,\"friends_count
\":1237,\"listed_count\":34,\"favourites_count\":30,\"statuses_count\":44052,\"created_at\":\"Fri Jun 01 15:05:49 +0000 2012\",\"utc_offset\":-14400,\"time_zone\":\"Eastern Time (US & Canada)\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/445184233460752385\\/RTBwpDuJ_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/445184233460752385\\/RTBwpDuJ_normal.jpeg\",\"profile_
banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/596541160\\/1408567298\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:21:13 +0000 2014\",\"id\":517333419173609472,\"id_str\":\"517333419173609472\",\"text\":\"Illegal Immigration: 70 Percent of Released Families Never Report Back To Federal Agents http:\\/\\/t.co\\/89DNq2IUFK #teaparty\",\"source\":\"\\u003ca href=\\\"http:\\/\\/bufferapp.com\\\" rel=\\\"nofollow\\\"\\u003eBuffer\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":6742412,\"id_str\":\"6742412\",\"name\":\"Jenny Beth Martin\",\"screen_name\":\"jennybethm\",\"location\":\"Atlanta, Ga\",\"ur
l\":\"http:\\/\\/jenuinejen.com\",\"description\":\"Co-Founder Tea Party Patriots; Tips, Tales, and Thoughts of Peach State Mom of Twins; Christian; Conservative; was Republican now American #tcot #teaparty #tgdn\",\"protected\":false,\"verified\":true,\"followers_count\":42320,\"friends_count\":23511,\"listed_count\":717,\"favourites_count\":570,\"statuses_count\":6266,\"created_at\":\"Mon Jun 11 14:32:10 +0000 2007\",\"utc_offset\":-14400,\"time_zone\":\"Eastern Time (US & Canada)\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"000000\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/95371301\\/new_fake_large_175-4.jpg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/95371301\\/new_fake_large_175-4.jpg\",\"profile_background_tile\":false,\"profile_link_color\":\"D60000\",\"profile_sidebar_border_color\":\"000000\"
,\"profile_sidebar_fill_color\":\"1C1939\",\"profile_text_color\":\"777777\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/2281265352\\/7e3d1pifsn0tjy895f5q_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/2281265352\\/7e3d1pifsn0tjy895f5q_normal.jpeg\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":4,\"favorite_count\":3,\"entities\":{\"hashtags\":[{\"text\":\"teaparty\",\"indices\":[112,121]}],\"trends\":[],\"urls\":[{\"url\":\"http:\\/\\/t.co\\/89DNq2IUFK\",\"expanded_url\":\"http:\\/\\/buff.ly\\/1nGdCzB\",\"display_url\":\"buff.ly\\/1nGdCzB\",\"indices\":[89,111]}],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"en\"},\
"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"teaparty\",\"indices\":[128,137]}],\"trends\":[],\"urls\":[{\"url\":\"http:\\/\\/t.co\\/89DNq2IUFK\",\"expanded_url\":\"http:\\/\\/buff.ly\\/1nGdCzB\",\"display_url\":\"buff.ly\\/1nGdCzB\",\"indices\":[105,127]}],\"user_mentions\":[{\"screen_name\":\"jennybethm\",\"name\":\"Jenny Beth Martin\",\"id\":6742412,\"id_str\":\"6742412\",\"indices\":[3,14]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_ms\":\"1412178008676\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180899864576,\"id_str\":\"517338180899864576\",\"text\":\"RT @ExtinctLana90: Je suis 100% d'accord pour mettre #YEnAVraimentMarreDesTypesQuiFontDesHashtag\\u00e0RallongeHeureusementQueTuMetsDesMajuscule\\u2026\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2245575537,\"id_str\":\"2245575537\",\"name\":\"krikristoophe\",\"screen_name\":\"krikristoophe\",\"location\":\"quelque part\",\"url\":null,\"description\":\"snapchat: christophe22470\\r\\ninstagram : @christophe22470 (trop d'imagination)\",\"protected\":false,\"verified\":false,\"followers_count\":393,\"friends_count\":370,\"listed_count\":1,\"favourites_count\":402,\"s
tatuses_count\":8458,\"created_at\":\"Thu Dec 26 23:27:14 +0000 2013\",\"utc_offset\":7200,\"time_zone\":\"Paris\",\"geo_enabled\":true,\"lang\":\"fr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"31A81C\",\"profile_sidebar_border_color\":\"000000\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/510763955930021888\\/RWzT-EhH_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/510763955930021888\\/RWzT-EhH_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/2245575537\\/140770
2870\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:31:10 +0000 2014\",\"id\":517335924968292353,\"id_str\":\"517335924968292353\",\"text\":\"Je suis 100% d'accord pour mettre #YEnAVraimentMarreDesTypesQuiFontDesHashtag\\u00e0RallongeHeureusementQueTuMetsDesMajusculesHein en TT :D\",\"source\":\"\\u003ca href=\\\"http:\\/\\/www.meltingmots.com\\\" rel=\\\"nofollow\\\"\\u003eExtinctLana90\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2593022185,\"id_str\":\"2593022185\",\"name\":\"Lana\",\"screen_name\":\"ExtinctLana90\",\"location\":\"\",\"url\":null,\"description\":\"Lana, pour vous servir. Ou pas.\\r\\n#T
eamInsomniaque.\",\"protected\":false,\"verified\":false,\"followers_count\":487,\"friends_count\":893,\"listed_count\":0,\"favourites_count\":1,\"statuses_count\":1829,\"created_at\":\"Sat Jun 28 12:14:50 +0000 2014\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"fr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/482916277862662145\\/iylCxj4N_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile
_images\\/482916277862662145\\/iylCxj4N_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/2593022185\\/1403971199\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":1,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"YEnAVraimentMarreDesTypesQuiFontDesHashtag\\u00e0RallongeHeureusementQueTuMetsDesMajusculesHein\",\"indices\":[35,125]}],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"fr\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"YEnAVraimentMarreDesTypesQuiFontDesHashtag\\u00e0RallongeHeureusementQueTuMetsDesMajusculesHein\",\"indices\":[54,140]}],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"Exti
nctLana90\",\"name\":\"Lana\",\"id\":2593022185,\"id_str\":\"2593022185\",\"indices\":[3,17]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"fr\",\"timestamp_ms\":\"1412178008683\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180912427008,\"id_str\":\"517338180912427008\",\"text\":\"RT @r1_follow1: \\u0628\\u064a\\u0639 \\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646 \\u062e\\u0644\\u064a\\u062c\\u064a\\u0646 \\u0644\\u0644\\u062a\\u0648\\u064a\\u062a\\u0631 \\u0648 \\u0625\\u0646\\u0633\\u062a\\u0642\\u0631\\u0627\\u0645 \\n\\n\\u0644\\u0644\\u062a\\u0648\\u0627\\u0635\\u0644 \\u0648\\u0627\\u062a\\u0633 \\u0627\\u0628 0596168454\\n\\nhttp:\\/\\/t.co\\/JzuAqNDCbV \\n 749\",\"source\":\"\\u003ca href=\\\"http:\\/\\/ffasr3.com\\/2\\\" rel=\\\"nofollow\\\"\\u003e\\u0627\\u0644\\u062a\\u0648\\u064a\\u062a\\u0631\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2650616521,\"id_str\":\"2650616521\",\"name\":\"quteba usma alani\",\"screen_name\":\"QutebaUsma\",\"location\":\"
\",\"url\":null,\"description\":null,\"protected\":false,\"verified\":false,\"followers_count\":9,\"friends_count\":40,\"listed_count\":0,\"favourites_count\":6,\"statuses_count\":2086,\"created_at\":\"Wed Jul 16 09:51:04 +0000 2014\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/abs.twimg.com\\/sticky\\/default_profile_images\\/default_profile_0_normal.png\",\"profile_image_url_https\":\"https:\\/\\/abs
.twimg.com\\/sticky\\/default_profile_images\\/default_profile_0_normal.png\",\"default_profile\":true,\"default_profile_image\":true,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:36:11 +0000 2014\",\"id\":517337187596390400,\"id_str\":\"517337187596390400\",\"text\":\"\\u0628\\u064a\\u0639 \\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646 \\u062e\\u0644\\u064a\\u062c\\u064a\\u0646 \\u0644\\u0644\\u062a\\u0648\\u064a\\u062a\\u0631 \\u0648 \\u0625\\u0646\\u0633\\u062a\\u0642\\u0631\\u0627\\u0645 \\n\\n\\u0644\\u0644\\u062a\\u0648\\u0627\\u0635\\u0644 \\u0648\\u0627\\u062a\\u0633 \\u0627\\u0628 0596168454\\n\\nhttp:\\/\\/t.co\\/JzuAqNDCbV \\n 749\",\"source\":\"\\u003ca href=\\\"http:\\/\\/ffasr3.com\\/a\\\" rel=\\\"nofollow\\\"\\u003etweet \\/a\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_i
d_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2800103305,\"id_str\":\"2800103305\",\"name\":\"\\u0628\\u064a\\u0639 \\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646\",\"screen_name\":\"r1_follow1\",\"location\":\"\",\"url\":null,\"description\":\"\\u200f\\u200f\\u0628\\u064a\\u0639 \\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646 \\u062d\\u0642\\u064a\\u0642\\u064a\\u0646 \\u062e\\u0644\\u064a\\u062c\\u064a\\u0646 \\u0644\\u0644\\u062a\\u0648\\u064a\\u062a\\u0631 \\u0648 \\u0627\\u0646\\u0633\\u062a\\u063a\\u0631\\u0627\\u0645 \\u0648 \\u0644\\u0627\\u064a\\u0643\\u0627\\u062a \\u062e\\u0644\\u064a\\u062c\\u064a\\u0646 \\u062b\\u0627\\u0628\\u062a\\u064a\\u0646 \\u0645\\u0639 \\u0636\\u0645\\u0627\\u0646 \\u0644\\u0643 \\u0627\\u0644\\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646 \\u0648\\u0627\\u062a\\u0633 \\u0627\\u0628 0596168454\",\"protected\":false,\"verified\":false,\"followers_count\":7704,\"
friends_count\":5500,\"listed_count\":4,\"favourites_count\":3,\"statuses_count\":3089,\"created_at\":\"Tue Sep 09 15:02:39 +0000 2014\",\"utc_offset\":3600,\"time_zone\":\"London\",\"geo_enabled\":false,\"lang\":\"fr\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/509428053143846912\\/gAG3VoKW_normal.png\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/509428053143846912\\/gAG3VoKW_normal.png\",\"profile_banner_url\":\
"https:\\/\\/pbs.twimg.com\\/profile_banners\\/2800103305\\/1410292101\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":1517,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[],\"media\":[{\"id\":515942230847021056,\"id_str\":\"515942230847021056\",\"indices\":[68,90],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"url\":\"http:\\/\\/t.co\\/JzuAqNDCbV\",\"display_url\":\"pic.twitter.com\\/JzuAqNDCbV\",\"expanded_url\":\"http:\\/\\/twitter.com\\/ilyass_h\\/status\\/515942234248998912\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"medium\":{\"w\":598,\
"h\":352,\"resize\":\"fit\"},\"small\":{\"w\":340,\"h\":200,\"resize\":\"fit\"}},\"source_status_id\":515942234248998912,\"source_status_id_str\":\"515942234248998912\"}]},\"extended_entities\":{\"media\":[{\"id\":515942230847021056,\"id_str\":\"515942230847021056\",\"indices\":[68,90],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"url\":\"http:\\/\\/t.co\\/JzuAqNDCbV\",\"display_url\":\"pic.twitter.com\\/JzuAqNDCbV\",\"expanded_url\":\"http:\\/\\/twitter.com\\/ilyass_h\\/status\\/515942234248998912\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"medium\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"small\":{\"w\":340,\"h\":200,\"resize\":\"fit\"}},\"source_status_id\":515942234248998912,\"source_status_id_str\":\"515942234248998912\"}]},\"favorited\":false,\"retweeted\":false,\"
possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"ar\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"r1_follow1\",\"name\":\"\\u0628\\u064a\\u0639 \\u0645\\u062a\\u0627\\u0628\\u0639\\u064a\\u0646\",\"id\":2800103305,\"id_str\":\"2800103305\",\"indices\":[3,14]}],\"symbols\":[],\"media\":[{\"id\":515942230847021056,\"id_str\":\"515942230847021056\",\"indices\":[84,106],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"url\":\"http:\\/\\/t.co\\/JzuAqNDCbV\",\"display_url\":\"pic.twitter.com\\/JzuAqNDCbV\",\"expanded_url\":\"http:\\/\\/twitter.com\\/ilyass_h\\/status\\/515942234248998912\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"medium\":{\"w\":598,\"h\":352,\"resize\":\"fit\
"},\"small\":{\"w\":340,\"h\":200,\"resize\":\"fit\"}},\"source_status_id\":515942234248998912,\"source_status_id_str\":\"515942234248998912\"}]},\"extended_entities\":{\"media\":[{\"id\":515942230847021056,\"id_str\":\"515942230847021056\",\"indices\":[84,106],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/Byj-ubJCYAAZchS.png\",\"url\":\"http:\\/\\/t.co\\/JzuAqNDCbV\",\"display_url\":\"pic.twitter.com\\/JzuAqNDCbV\",\"expanded_url\":\"http:\\/\\/twitter.com\\/ilyass_h\\/status\\/515942234248998912\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"medium\":{\"w\":598,\"h\":352,\"resize\":\"fit\"},\"small\":{\"w\":340,\"h\":200,\"resize\":\"fit\"}},\"source_status_id\":515942234248998912,\"source_status_id_str\":\"515942234248998912\"}]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":fals
e,\"filter_level\":\"medium\",\"lang\":\"ar\",\"timestamp_ms\":\"1412178008686\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180891467776,\"id_str\":\"517338180891467776\",\"text\":\"RT @K31L4_: @oliviazecchin waaaaaaa que decis personaje jajajaja\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/android\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Android\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":1473191491,\"id_str\":\"1473191491\",\"name\":\"Oli\",\"screen_name\":\"oliviazecchin\",\"location\":\"\",\"url\":null,\"description\":\"Y la verdad, no hay cosa peor, \\nque vivir sin pasi\\u00f3n..\",\"protected\":false,\"verified\":false,\"followers_count\":360,\"friends_count\":325,\"listed_count\":0,\"favourites_count\":1688,\"statuses_count\":11441,\"created_at\":\"Fri May 31 21:55:06 +0000 2013\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\
":true,\"lang\":\"es\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"ACDED6\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme18\\/bg.gif\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme18\\/bg.gif\",\"profile_background_tile\":false,\"profile_link_color\":\"038543\",\"profile_sidebar_border_color\":\"EEEEEE\",\"profile_sidebar_fill_color\":\"F6F6F6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/509118978350804992\\/Topjl7PV_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/509118978350804992\\/Topjl7PV_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/1473191491\\/1410377782\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notification
s\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:27:42 +0000 2014\",\"id\":517335050082205696,\"id_str\":\"517335050082205696\",\"text\":\"@oliviazecchin waaaaaaa que decis personaje jajajaja\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\\" rel=\\\"nofollow\\\"\\u003eTwitter Web Client\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":517333590489964545,\"in_reply_to_status_id_str\":\"517333590489964545\",\"in_reply_to_user_id\":1473191491,\"in_reply_to_user_id_str\":\"1473191491\",\"in_reply_to_screen_name\":\"oliviazecchin\",\"user\":{\"id\":917988457,\"id_str\":\"917988457\",\"name\":\"cronopio redondito\",\"screen_name\":\"K31L4_\",\"location\":\"Carlos Alberto Garc\\u00eda Moreno\",\"url\":null,\"description\":\"\\u2119\\u211d\\u2655 y sus Redonditos de Ricota y AP en algunos ciclos solares. Club Atl\\u00e9tico Rosario Central. Mi vieja cri\\u00f3 un idiota de coraz\\
u00f3n lun\\u00e1tico.\",\"protected\":false,\"verified\":false,\"followers_count\":97,\"friends_count\":119,\"listed_count\":0,\"favourites_count\":149,\"statuses_count\":4060,\"created_at\":\"Thu Nov 01 01:01:39 +0000 2012\",\"utc_offset\":-10800,\"time_zone\":\"Santiago\",\"geo_enabled\":false,\"lang\":\"es\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"131516\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/469629734930771968\\/-8Z_79Q1.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/469629734930771968\\/-8Z_79Q1.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"42171A\",\"profile_sidebar_border_color\":\"000000\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/478290344656650240\\/cYsZS_tY_norm
al.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/478290344656650240\\/cYsZS_tY_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/917988457\\/1408206021\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":1,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"oliviazecchin\",\"name\":\"Oli\",\"id\":1473191491,\"id_str\":\"1473191491\",\"indices\":[0,14]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"es\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"K31L4_\",\"name\":\"cronopio redondito\",\"id\":917988457,\"id_str\":\"91798
8457\",\"indices\":[3,10]},{\"screen_name\":\"oliviazecchin\",\"name\":\"Oli\",\"id\":1473191491,\"id_str\":\"1473191491\",\"indices\":[12,26]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"es\",\"timestamp_ms\":\"1412178008684\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180912443392,\"id_str\":\"517338180912443392\",\"text\":\"@BD_Lay I Knew It \\ud83d\\ude02\\ud83d\\ude2d\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/android\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Android\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":517335209826848769,\"in_reply_to_status_id_str\":\"517335209826848769\",\"in_reply_to_user_id\":311668757,\"in_reply_to_user_id_str\":\"311668757\",\"in_reply_to_screen_name\":\"BD_Lay\",\"user\":{\"id\":568404382,\"id_str\":\"568404382\",\"name\":\"NUT WORLD BITCH\\u2757\",\"screen_name\":\"Ballout_4250\",\"location\":\"Welcome To Nut World\\u2757 4\\u20e32\\u20e35\\u20e30\\u20e3\",\"url\":null,\"description\":\"Big Ballout FRM The A\\u2757 Free All The Guys \\u2757R.I.P. Hoddy & Nut\\u2757.. Love Viniya Life \\u2757\\ufe0f\\u2764\\ufe0f\",\"protected\":false,\"verified\":false,\"followers_count\":963,\"friends_count\":90
7,\"listed_count\":2,\"favourites_count\":9224,\"statuses_count\":56249,\"created_at\":\"Tue May 01 17:52:53 +0000 2012\",\"utc_offset\":-10800,\"time_zone\":\"Atlantic Time (Canada)\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/378800000039624360\\/ed7ce9d140503e73711e6d71fe6fce1c.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/378800000039624360\\/ed7ce9d140503e73711e6d71fe6fce1c.jpeg\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"000000\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/508187491305009152\\/uE0eB6zG_normal.jpeg\",\"profile_image_url_https\":\"ht
tps:\\/\\/pbs.twimg.com\\/profile_images\\/508187491305009152\\/uE0eB6zG_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/568404382\\/1409189327\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"BD_Lay\",\"name\":\"sheemagang \",\"id\":311668757,\"id_str\":\"311668757\",\"indices\":[0,7]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_ms\":\"1412178008671\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180886872064,\"id_str\":\"517338180886872064\",\"text\":\"\\u5a18\\u3002\\u30b3\\u30f3\\u306e\\u65e5\\u3060\\u3063\\u305f\\u306e\\u306b\\u4f55\\u6545\\u304b\\u8133\\u5185\\u3067\\u306f\\u30d9\\u30ea\\u304c\\u6d41\\u308c\\u3066\\u304a\\u308b\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":132108937,\"id_str\":\"132108937\",\"name\":\"\\u72c2\\u30cf\\u30e0\\u3061\\u3072\\u308d'14\",\"screen_name\":\"chii_hs\",\"location\":\"\\u3055\\u3093\\u308a\\u304a\\u3074\\u3085\\u30fc\\u308d\\u3089\\u3093\\u3069\",\"url\":null,\"description\":\"\\u6b4c\\u3046\\u3053\\u3068\\u3068\\u4f5c\\u308b\\u3053\\u3068\\u304c\\u597d\\u304d\\u3067\\u30
59\\u3002\\u30e9\\u30fc\\u30e1\\u30f3*\\u30b5\\u30f3\\u30ea\\u30aa*\\u30b7\\u30ca\\u30e2\\u30ed\\u30fc\\u30eb*\\u8584\\u685c\\u9b3c*\\u304a\\u6d0b\\u670d*\\u30cf\\u30ed\\u30d7\\u30ed*\\u30e2\\u30fc\\u30cb\\u30f3\\u30b0\\u5a18\\u3002*\",\"protected\":false,\"verified\":false,\"followers_count\":388,\"friends_count\":391,\"listed_count\":15,\"favourites_count\":2259,\"statuses_count\":40945,\"created_at\":\"Mon Apr 12 09:04:44 +0000 2010\",\"utc_offset\":32400,\"time_zone\":\"Tokyo\",\"geo_enabled\":true,\"lang\":\"ja\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"1A1B1F\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme9\\/bg.gif\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme9\\/bg.gif\",\"profile_background_tile\":false,\"profile_link_color\":\"2FC2EF\",\"profile_sidebar_border_color\":\"181A1E\",\"profile_sidebar_fill_color\":\"252429\",\"profile_text_color\
":\"666666\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/458234331832410112\\/UW0e7cCy_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/458234331832410112\\/UW0e7cCy_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/132108937\\/1395030721\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"ja\",\"timestamp_ms\":\"1412178008664\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180904030210,\"id_str\":\"517338180904030210\",\"text\":\"RT @V8888S: \\u0645\\u0633\\u0627\\u0621 \\u0627\\u0644\\u0643\\u0644\\u0645\\u0629 \\u0627\\u0644\\u062d\\u0644\\u0648\\u0629 \\u0645\\u0633\\u0627\\u0621 \\u0627\\u0644\\u062d\\u0628 \\u0628\\u0644\\u064a\\u0627 \\u062d\\u062f\\u0648\\u062f \\u0645\\u0633\\u0627\\u0621 \\u064a\\u062d\\u062a\\u0648\\u064a\\u0647 \\u0627\\u0644\\u0643\\u0648\\u0646 \\u0644\\u0623\\u0646\\u0643 \\u0623\\u0646\\u062a \\u0641\\u064a\\u0647 \\u0645\\u0648\\u062c\\u0648\\u062f \\u0645\\u0633\\u0627\\u0626\\u064a \\u0627\\u0646\\u062a\\u0650 \\u2764\\ufe0f\\ud83d\\udc8b\\ud83c\\udf39\",\"source\":\"\\u003ca href=\\\"http:\\/\\/vsbelle.net\\/120\\/\\\" rel=\\\"nofollow\\\"\\u003ehttp:\\/\\/vsbelle.net\\/120\\/\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_r
eply_to_screen_name\":null,\"user\":{\"id\":2254043439,\"id_str\":\"2254043439\",\"name\":\"Camilla\\r Darcey\\r\",\"screen_name\":\"a_lndem00\",\"location\":\"Texas\\r\",\"url\":null,\"description\":\"\\u0644\\u0644\\u0635\\u062f\\u0627\\u0642\\u0629 (\\u0627\\u0644\\u062e\\u0627\\u0635)\\n\\n\\u0627\\u0644\\u0631\\u064a\\u0627\\u0636\",\"protected\":false,\"verified\":false,\"followers_count\":81,\"friends_count\":521,\"listed_count\":0,\"favourites_count\":2339,\"statuses_count\":24523,\"created_at\":\"Tue Dec 31 11:32:31 +0000 2013\",\"utc_offset\":-25200,\"time_zone\":\"Pacific Time (US & Canada)\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"141414\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme7\\/bg.gif\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme7\\/bg.gif\",\"profile_background_tile\":true,\"profile_link_co
lor\":\"942C2C\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"F3F3F3\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/430389252866592768\\/SWbXjVTn_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/430389252866592768\\/SWbXjVTn_normal.jpeg\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 15:32:09 +0000 2014\",\"id\":517336173060968448,\"id_str\":\"517336173060968448\",\"text\":\"\\u0645\\u0633\\u0627\\u0621 \\u0627\\u0644\\u0643\\u0644\\u0645\\u0629 \\u0627\\u0644\\u062d\\u0644\\u0648\\u0629 \\u0645\\u0633\\u0627\\u0621 \\u0627\\u0644\\u062d\\u0628 \\u0628\\u0644\\u064a\\u0627 \\u062d\\u062f\\u0648\\u062f \\u0645\\u0633\\
u0627\\u0621 \\u064a\\u062d\\u062a\\u0648\\u064a\\u0647 \\u0627\\u0644\\u0643\\u0648\\u0646 \\u0644\\u0623\\u0646\\u0643 \\u0623\\u0646\\u062a \\u0641\\u064a\\u0647 \\u0645\\u0648\\u062c\\u0648\\u062f \\u0645\\u0633\\u0627\\u0626\\u064a \\u0627\\u0646\\u062a\\u0650 \\u2764\\ufe0f\\ud83d\\udc8b\\ud83c\\udf39\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2740802604,\"id_str\":\"2740802604\",\"name\":\"\\u0631\\u062c\\u0644 \\u0645\\u0632\\u0622\\u062c\\u064a\",\"screen_name\":\"V8888S\",\"location\":\"\",\"url\":null,\"description\":\"\\u0627\\u0646\\u0627 \\u0631\\u062c\\u0644 \\u0645\\u0632\\u0627\\u062c\\u064a \\u0630\\u0622\\u062a \\u0645\\u0641\\u0631\\u062f\\u0622\\u062a \\u0639\\u0646
\\u064a\\u062f\\u06be\\u06c1\\u064e \\u0644\\u0633\\u062a \\u0645\\u062c\\u0628\\u0648\\u0631 \\u0639\\u0644\\u0649 \\u062a\\u0628\\u0631\\u064a\\u064e\\u0631 \\u0645\\u0648\\u0622\\u0642\\u0641\\u064a \\u0644\\u0645\\u0646\\u064e \\u064a\\u0633\\u064a\\u0621 \\u0622\\u0644\\u0638\\u0646 \\u0622\\u062a\\u062d\\u062f\\u062b \\u0628\\u0639\\u0641\\u0648\\u064a\\u06be\\u06c1\\u064e \\u0622\\u0636\\u062d\\u0643\\u0643 \\u0628\\u0634\\u062f\\u06be\\u06c1\\u064e \\u0623\\u063a\\u0627\\u0631 \\u0628\\u062c\\u0646\\u0648\\u0646 \\u0648\\u0623\\u0639\\u0634\\u0642 \\u0628\\u062e\\u062c\\u0644 .. \\u0623\\u0646\\u062a\\u0645\\u064a \\u0644\\u0644\\u0647\\u0644\\u0627\\u0644\",\"protected\":false,\"verified\":false,\"followers_count\":9443,\"friends_count\":153,\"listed_count\":0,\"favourites_count\":81,\"statuses_count\":3183,\"created_at\":\"Mon Aug 18 00:21:00 +0000 2014\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"ar\",\"contributors_enabled\":false,\"is_transl
ator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/501402712164933633\\/jfgUf6cZ_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/501402712164933633\\/jfgUf6cZ_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/2740802604\\/1412092338\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contrib
utors\":null,\"retweet_count\":671,\"favorite_count\":1,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"low\",\"lang\":\"ar\"},\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"V8888S\",\"name\":\"\\u0631\\u062c\\u0644 \\u0645\\u0632\\u0622\\u062c\\u064a\",\"id\":2740802604,\"id_str\":\"2740802604\",\"indices\":[3,10]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"ar\",\"timestamp_ms\":\"1412178008681\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180891443201,\"id_str\":\"517338180891443201\",\"text\":\"\\u0622\\u0628\\u0644\\u063a \\u0645\\u0646 \\u0627\\u0644\\u0643\\u0644\\u0627\\u0645 [ \\u0627\\u0644\\u0644\\u0647\\u0645 \\u0623\\u0646\\u0639\\u0645 \\u0639\\u0644\\u064a\\u0646\\u0627 \\u0628\\u062d\\u0645\\u062f\\u0643 \\u0648 \\u0631\\u0636\\u0627\\u0643 ] #\\u063a\\u0631\\u062f_\\u0628\\u0635\\u0648\\u0631\\u0629\",\"source\":\"\\u003ca href=\\\"http:\\/\\/batross.com\\\" rel=\\\"nofollow\\\"\\u003e\\u0628\\u0627\\u0644\\u062e\\u064a\\u0631 \\u063a\\u0631\\u062f\\u0644\\u064a\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":603454115,\"id_str\":\"603454115\",\"name\":\"\\u0627\\u0644\\u0643\\u0627\\u0633\\u06312006\",\"screen_name\":\"DKL2006\",\"location\":\"\\u0627\\u0644\\u0631\\u0
64a\\u0627\\u0636\",\"url\":null,\"description\":\"\\u200f\\u200f\\u0645\\u0648\\u0638\\u0641 (\\u0627\\u062d\\u0628 \\u0627\\u0644\\u062d\\u064a\\u0627\\u0629 \\u0648\\u0627\\u0646 \\u063a\\u062f\\u0631\\u0648 \\u0628\\u064a \\u0627\\u0644\\u0628\\u0634\\u0631)\",\"protected\":false,\"verified\":false,\"followers_count\":864,\"friends_count\":1997,\"listed_count\":0,\"favourites_count\":21,\"statuses_count\":5489,\"created_at\":\"Sat Jun 09 09:21:16 +0000 2012\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":true,\"lang\":\"ar\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",
\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/514354297622892545\\/wnTPiDj6_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/514354297622892545\\/wnTPiDj6_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/603454115\\/1411467703\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[{\"text\":\"\\u063a\\u0631\\u062f_\\u0628\\u0635\\u0648\\u0631\\u0629\",\"indices\":[49,59]}],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"ar\",\"timestamp_ms\":\"1412178008664\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180912443393,\"id_str\":\"517338180912443393\",\"text\":\"@jackspotlight amo os vil\\u00f5es, eles s\\u00e3o mais bonitos e gostosos hhahahhhaa\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/android\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Android\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":517337999437078529,\"in_reply_to_status_id_str\":\"517337999437078529\",\"in_reply_to_user_id\":109532411,\"in_reply_to_user_id_str\":\"109532411\",\"in_reply_to_screen_name\":\"jackspotlight\",\"user\":{\"id\":872394073,\"id_str\":\"872394073\",\"name\":\"styles\",\"screen_name\":\"kylizstyles\",\"location\":\"coxa do harry\",\"url\":\"http:\\/\\/nobody-knows-you-the-way-i-do.tumblr.com\",\"description\":\"jenner's\\/ kardashian's e viados \\u2661 wwa tour \\u2661\",\"protected\":false,\"verified\":false,\"followers_count\":1834,\"friends_count\":1979,\"listed_count\":1,\"favourites_
count\":3815,\"statuses_count\":60978,\"created_at\":\"Wed Oct 10 19:39:49 +0000 2012\",\"utc_offset\":-7200,\"time_zone\":\"Mid-Atlantic\",\"geo_enabled\":true,\"lang\":\"pt\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"FFFFFF\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/469603339643015168\\/scY2ldKq.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/469603339643015168\\/scY2ldKq.jpeg\",\"profile_background_tile\":false,\"profile_link_color\":\"ABB8C2\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/513517169775501312\\/1Emb0kk2_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/513517169775501312\\/1Emb0kk2_normal.jpeg\",\"profi
le_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/872394073\\/1411510767\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"jackspotlight\",\"name\":\"camila\",\"id\":109532411,\"id_str\":\"109532411\",\"indices\":[0,14]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"pt\",\"timestamp_ms\":\"1412178008671\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180895649793,\"id_str\":\"517338180895649793\",\"text\":\"Fat chats - The good, the bad and the ugly comments: Study analyzes how people chat about weight on different socia\\u2026 http:\\/\\/t.co\\/kIOa0ebpn7\",\"source\":\"\\u003ca href=\\\"http:\\/\\/www.reddit.com\\/\\\" rel=\\\"nofollow\\\"\\u003eReddit RSS\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2577635288,\"id_str\":\"2577635288\",\"name\":\"Reddit Science\",\"screen_name\":\"Science_Reddit\",\"location\":\"\",\"url\":\"http:\\/\\/www.reddit.com\\/r\\/science\\/new\\/\",\"description\":\"Newest submissions from \\/r\\/science with original links.\",\"protected\":false,\"verified\":false,\"followers_count\":272,\"friends_count\":12,\"listed_count\":6,\"favourites_count\":0,\"statuses_c
ount\":9549,\"created_at\":\"Thu Jun 19 23:11:01 +0000 2014\",\"utc_offset\":10800,\"time_zone\":\"Tallinn\",\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/480229204919742465\\/DHLw_QAc_normal.png\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/480229204919742465\\/DHLw_QAc_normal.png\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent
\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[{\"url\":\"http:\\/\\/t.co\\/kIOa0ebpn7\",\"expanded_url\":\"http:\\/\\/www.springer.com\\/gp\\/about-springer\\/media\\/springer-select\\/fat-chats-the-good-the-bad-and-the-ugly-comments\\/35816\",\"display_url\":\"springer.com\\/gp\\/about-sprin\\u2026\",\"indices\":[117,139]}],\"user_mentions\":[],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_ms\":\"1412178008665\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180908249088,\"id_str\":\"517338180908249088\",\"text\":\"When you truly forgive if ever given the opportunity to get even you won't even new worried about it you just move on\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/android\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Android\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":268692270,\"id_str\":\"268692270\",\"name\":\"Mz. Bossy \",\"screen_name\":\"MzBarbieondat\",\"location\":\"Da Pharmacy \",\"url\":null,\"description\":\"I am that bitch you either love to hate or hate to love either way I'm still dat bitch! #ObeyReality #UGLYGang #CaponeNation #WeMoBBin\",\"protected\":false,\"verified\":false,\"followers_count\":1386,\"friends_count\":1990,\"listed_count\":1,\"favourit
es_count\":457,\"statuses_count\":9563,\"created_at\":\"Sat Mar 19 08:18:08 +0000 2011\",\"utc_offset\":-18000,\"time_zone\":\"Central Time (US & Canada)\",\"geo_enabled\":true,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/730188034\\/989425a06f0c5e65ce6b68c77c2551a1.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/730188034\\/989425a06f0c5e65ce6b68c77c2551a1.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/465219107516723200\\/DHqepczu_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/465219
107516723200\\/DHqepczu_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/268692270\\/1411370160\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":{\"type\":\"Point\",\"coordinates\":[34.923717,-88.496280]},\"coordinates\":{\"type\":\"Point\",\"coordinates\":[-88.496280,34.923717]},\"place\":{\"id\":\"8c00285181cdb467\",\"url\":\"https:\\/\\/api.twitter.com\\/1.1\\/geo\\/id\\/8c00285181cdb467.json\",\"place_type\":\"city\",\"name\":\"Corinth\",\"full_name\":\"Corinth, MS\",\"country_code\":\"US\",\"country\":\"United States\",\"bounding_box\":{\"type\":\"Polygon\",\"coordinates\":[[[-88.576522,34.901195],[-88.576522,34.964867],[-88.4694382,34.964867],[-88.4694382,34.901195]]]},\"attributes\":{}},\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[],\"symbols\":[]},\"favorited
\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_ms\":\"1412178008669\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180882690049,\"id_str\":\"517338180882690049\",\"text\":\"I complimented Susie McBeth\\u2019s page at @aboutdotme! Check it out: http:\\/\\/t.co\\/KYa7ly9oyo @susiemcbeth\",\"source\":\"\\u003ca href=\\\"https:\\/\\/dev.twitter.com\\/docs\\/tfw\\\" rel=\\\"nofollow\\\"\\u003eTwitter for Websites\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":2331974365,\"id_str\":\"2331974365\",\"name\":\"Miranda N. Prather\",\"screen_name\":\"MirandaNPrather\",\"location\":\"Middletown, MD\",\"url\":\"http:\\/\\/www.facebook.com\\/blueblueseaottb\",\"description\":\"Writer, horselover, Born in TX, but got my soul in NOLA\\r\\n\\r\\nArea Coordinator for: http:\\/\\/www.grantstoyou.org\\/chapterdisplay.php?id=104\",\"protected\":false,\"verified\":false,\"followers_
count\":94,\"friends_count\":266,\"listed_count\":2,\"favourites_count\":481,\"statuses_count\":616,\"created_at\":\"Fri Feb 07 14:44:57 +0000 2014\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_background_images\\/512644583595835392\\/U80V8y5D.jpeg\",\"profile_background_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_background_images\\/512644583595835392\\/U80V8y5D.jpeg\",\"profile_background_tile\":true,\"profile_link_color\":\"333333\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"0084B4\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/512645616459980800\\/ZOD6qc7y_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/51
2645616459980800\\/ZOD6qc7y_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/2331974365\\/1411059107\",\"default_profile\":false,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[{\"url\":\"http:\\/\\/t.co\\/KYa7ly9oyo\",\"expanded_url\":\"http:\\/\\/about.me\\/susiemcbeth\",\"display_url\":\"about.me\\/susiemcbeth\",\"indices\":[65,87]}],\"user_mentions\":[{\"screen_name\":\"aboutdotme\",\"name\":\"about.me\",\"id\":115304519,\"id_str\":\"115304519\",\"indices\":[38,49]},{\"screen_name\":\"susiemcbeth\",\"name\":\"Susie McBeth\",\"id\":22052388,\"id_str\":\"22052388\",\"indices\":[88,100]}],\"symbols\":[]},\"favorited\":false,\"retweeted\":false,\"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\",\"timestamp_m
s\":\"1412178008664\"}",
- "{\"created_at\":\"Wed Oct 01 15:40:08 +0000 2014\",\"id\":517338180904026113,\"id_str\":\"517338180904026113\",\"text\":\"RT @danmfc90: \\u201c@theawayfans: Roma fans celebrating their goal at The Etihad last night http:\\/\\/t.co\\/XrRhhiQE0A\\u201d lad in the Millwall hat ther\\u2026\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":473835254,\"id_str\":\"473835254\",\"name\":\"harry weaver\",\"screen_name\":\"1HarryWeaver1\",\"location\":\"\",\"url\":null,\"description\":null,\"protected\":false,\"verified\":false,\"followers_count\":490,\"friends_count\":1542,\"listed_count\":0,\"favourites_count\":176,\"statuses_count\":2109,\"created_at\":\"Wed Jan 25 11:28:00 +0000 2012\",\"utc_offse
t\":null,\"time_zone\":null,\"geo_enabled\":true,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/486841189346582529\\/VFZXNXYx_normal.jpeg\",\"profile_image_url_https\":\"https:\\/\\/pbs.twimg.com\\/profile_images\\/486841189346582529\\/VFZXNXYx_normal.jpeg\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null
,\"contributors\":null,\"retweeted_status\":{\"created_at\":\"Wed Oct 01 12:55:49 +0000 2014\",\"id\":517296830582108161,\"id_str\":\"517296830582108161\",\"text\":\"\\u201c@theawayfans: Roma fans celebrating their goal at The Etihad last night http:\\/\\/t.co\\/XrRhhiQE0A\\u201d lad in the Millwall hat there haha \\ud83d\\udc4f\",\"source\":\"\\u003ca href=\\\"http:\\/\\/twitter.com\\/download\\/iphone\\\" rel=\\\"nofollow\\\"\\u003eTwitter for iPhone\\u003c\\/a\\u003e\",\"truncated\":false,\"in_reply_to_status_id\":517287910224461824,\"in_reply_to_status_id_str\":\"517287910224461824\",\"in_reply_to_user_id\":1107326988,\"in_reply_to_user_id_str\":\"1107326988\",\"in_reply_to_screen_name\":\"theawayfans\",\"user\":{\"id\":258893426,\"id_str\":\"258893426\",\"name\":\"Dan Pullen\",\"screen_name\":\"danmfc90\",\"location\":\"Essex\",\"url\":null,\"description\":\"Im an opinionated baby eating wife stabbing millwall fan who enjoys long drunk train journeys up north.. Will probably of
fend you tbh... Dont believe the hype\",\"protected\":false,\"verified\":false,\"followers_count\":1537,\"friends_count\":1325,\"listed_count\":6,\"favourites_count\":17583,\"statuses_count\":55539,\"created_at\":\"Mon Feb 28 18:48:57 +0000 2011\",\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":true,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_image_url_https\":\"https:\\/\\/abs.twimg.com\\/images\\/themes\\/theme1\\/bg.png\",\"profile_background_tile\":false,\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"profile_image_url\":\"http:\\/\\/pbs.twimg.com\\/profile_images\\/512530957979881473\\/waAvdqKb_normal.jpeg\",\"profile_image_url_https\":\"https:\
\/\\/pbs.twimg.com\\/profile_images\\/512530957979881473\\/waAvdqKb_normal.jpeg\",\"profile_banner_url\":\"https:\\/\\/pbs.twimg.com\\/profile_banners\\/258893426\\/1408263439\",\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":4,\"favorite_count\":4,\"entities\":{\"hashtags\":[],\"trends\":[],\"urls\":[],\"user_mentions\":[{\"screen_name\":\"theawayfans\",\"name\":\"The Away Fans\",\"id\":1107326988,\"id_str\":\"1107326988\",\"indices\":[1,13]}],\"symbols\":[],\"media\":[{\"id\":517287908084948992,\"id_str\":\"517287908084948992\",\"indices\":[73,95],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3GnJgCIAAK2O8.jpg\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3GnJgCIAAK2O8.jpg\",\"url\":\"http:\\/\\/t.co\\/XrRhhiQE0A\",\"display_url\":\"pic.twitter.com\\/XrRhhiQE0A\",\"expanded_url\":\"http:\\/\
\/twitter.com\\/theawayfans\\/status\\/517287910224461824\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"large\":{\"w\":952,\"h\":594,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":340,\"h\":212,\"resize\":\"fit\"},\"medium\":{\"w\":600,\"h\":374,\"resize\":\"fit\"}},\"source_status_id\":517287910224461824,\"source_status_id_str\":\"517287910224461824\"}]},\"extended_entities\":{\"media\":[{\"id\":517287908084948992,\"id_str\":\"517287908084948992\",\"indices\":[73,95],\"media_url\":\"http:\\/\\/pbs.twimg.com\\/media\\/By3GnJgCIAAK2O8.jpg\",\"media_url_https\":\"https:\\/\\/pbs.twimg.com\\/media\\/By3GnJgCIAAK2O8.jpg\",\"url\":\"http:\\/\\/t.co\\/XrRhhiQE0A\",\"display_url\":\"pic.twitter.com\\/XrRhhiQE0A\",\"expanded_url\":\"http:\\/\\/twitter.com\\/theawayfans\\/status\\/517287910224461824\\/photo\\/1\",\"type\":\"photo\",\"sizes\":{\"large\":{\"w\":952,\"h\":594,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"smal
l\":{\"w\":340,\"h\":212,\"resize\":\"fit\"},\"medium\":{\"w\":600,\"h\":374,\"resize\":\"fit\"}},\"source_status_id\":51728791022446182
<TRUNCATED>
[39/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
deleted file mode 100644
index 4fe356b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ /dev/null
@@ -1,658 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.common.functions.RichReduceFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
-import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
-import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
-import org.apache.flink.streaming.api.invokable.operator.WindowGroupReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.WindowReduceInvokable;
-import org.apache.flink.streaming.api.windowing.helper.Time;
-import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
-import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
-
-/**
- * A {@link WindowedDataStream} represents a data stream that has been divided
- * into windows (predefined chunks). User defined function such as
- * {@link #reduce(ReduceFunction)}, {@link #reduceGroup(GroupReduceFunction)} or
- * aggregations can be applied to the windows.
- *
- * @param <OUT>
- * The output type of the {@link WindowedDataStream}
- */
-public class WindowedDataStream<OUT> {
-
- protected DataStream<OUT> dataStream;
- protected boolean isGrouped;
- protected boolean allCentral;
- protected KeySelector<OUT, ?> keySelector;
-
- protected List<WindowingHelper<OUT>> triggerHelpers;
- protected List<WindowingHelper<OUT>> evictionHelpers;
-
- protected LinkedList<TriggerPolicy<OUT>> userTriggers;
- protected LinkedList<EvictionPolicy<OUT>> userEvicters;
-
- protected WindowedDataStream(DataStream<OUT> dataStream, WindowingHelper<OUT>... policyHelpers) {
- this.dataStream = dataStream.copy();
- this.triggerHelpers = new ArrayList<WindowingHelper<OUT>>();
- for (WindowingHelper<OUT> helper : policyHelpers) {
- this.triggerHelpers.add(helper);
- }
-
- if (dataStream instanceof GroupedDataStream) {
- this.isGrouped = true;
- this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
- // set all policies distributed
- this.allCentral = false;
-
- } else {
- this.isGrouped = false;
- // set all policies central
- this.allCentral = true;
- }
- }
-
- protected WindowedDataStream(DataStream<OUT> dataStream, List<TriggerPolicy<OUT>> triggers,
- List<EvictionPolicy<OUT>> evicters) {
- this.dataStream = dataStream.copy();
-
- if (triggers != null) {
- this.userTriggers = new LinkedList<TriggerPolicy<OUT>>();
- this.userTriggers.addAll(triggers);
- }
-
- if (evicters != null) {
- this.userEvicters = new LinkedList<EvictionPolicy<OUT>>();
- this.userEvicters.addAll(evicters);
- }
-
- if (dataStream instanceof GroupedDataStream) {
- this.isGrouped = true;
- this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
- // set all policies distributed
- this.allCentral = false;
-
- } else {
- this.isGrouped = false;
- // set all policies central
- this.allCentral = true;
- }
- }
-
- protected WindowedDataStream(WindowedDataStream<OUT> windowedDataStream) {
- this.dataStream = windowedDataStream.dataStream.copy();
- this.isGrouped = windowedDataStream.isGrouped;
- this.keySelector = windowedDataStream.keySelector;
- this.triggerHelpers = windowedDataStream.triggerHelpers;
- this.evictionHelpers = windowedDataStream.evictionHelpers;
- this.userTriggers = windowedDataStream.userTriggers;
- this.userEvicters = windowedDataStream.userEvicters;
- this.allCentral = windowedDataStream.allCentral;
- }
-
- public <F> F clean(F f) {
- return dataStream.clean(f);
- }
-
- /**
- * Defines the slide size (trigger frequency) for the windowed data stream.
- * This controls how often the user defined function will be triggered on
- * the window. </br></br> For example to get a window of 5 elements with a
- * slide of 2 seconds use: </br></br>
- * {@code ds.window(Count.of(5)).every(Time.of(2,TimeUnit.SECONDS))}
- * </br></br> The user function in this case will be called on the 5 most
- * recent elements every 2 seconds
- *
- * @param policyHelpers
- * The policies that define the triggering frequency
- *
- * @return The windowed data stream with triggering set
- */
- @SuppressWarnings({ "unchecked", "rawtypes" })
- public WindowedDataStream<OUT> every(WindowingHelper... policyHelpers) {
- WindowedDataStream<OUT> ret = this.copy();
- if (ret.evictionHelpers == null) {
- ret.evictionHelpers = ret.triggerHelpers;
- ret.triggerHelpers = new ArrayList<WindowingHelper<OUT>>();
- }
- for (WindowingHelper<OUT> helper : policyHelpers) {
- ret.triggerHelpers.add(helper);
- }
- return ret;
- }
-
- /**
- * Groups the elements of the {@link WindowedDataStream} by the given key
- * positions. The window sizes (evictions) and slide sizes (triggers) will
- * be calculated on the whole stream (in a central fashion), but the user
- * defined functions will be applied on a per group basis. </br></br> To get
- * windows and triggers on a per group basis apply the
- * {@link DataStream#window} operator on an already grouped data stream.
- *
- * @param fields
- * The position of the fields to group by.
- * @return The grouped {@link WindowedDataStream}
- */
- public WindowedDataStream<OUT> groupBy(int... fields) {
- WindowedDataStream<OUT> ret = this.copy();
- ret.dataStream = ret.dataStream.groupBy(fields);
- ret.isGrouped = true;
- ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
- return ret;
- }
-
- /**
- * Groups the elements of the {@link WindowedDataStream} by the given field
- * expressions. The window sizes (evictions) and slide sizes (triggers) will
- * be calculated on the whole stream (in a central fashion), but the user
- * defined functions will be applied on a per group basis. </br></br> To get
- * windows and triggers on a per group basis apply the
- * {@link DataStream#window} operator on an already grouped data stream.
- * </br></br> A field expression is either the name of a public field or a
- * getter method with parentheses of the stream's underlying type. A dot can
- * be used to drill down into objects, as in
- * {@code "field1.getInnerField2()" }.
- *
- * @param fields
- * The fields to group by
- * @return The grouped {@link WindowedDataStream}
- */
- public WindowedDataStream<OUT> groupBy(String... fields) {
- WindowedDataStream<OUT> ret = this.copy();
- ret.dataStream = ret.dataStream.groupBy(fields);
- ret.isGrouped = true;
- ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
- return ret;
- }
-
- /**
- * Groups the elements of the {@link WindowedDataStream} using the given
- * {@link KeySelector}. The window sizes (evictions) and slide sizes
- * (triggers) will be calculated on the whole stream (in a central fashion),
- * but the user defined functions will be applied on a per group basis.
- * </br></br> To get windows and triggers on a per group basis apply the
- * {@link DataStream#window} operator on an already grouped data stream.
- *
- * @param keySelector
- * The keySelector used to extract the key for grouping.
- * @return The grouped {@link WindowedDataStream}
- */
- public WindowedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
- WindowedDataStream<OUT> ret = this.copy();
- ret.dataStream = ret.dataStream.groupBy(keySelector);
- ret.isGrouped = true;
- ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
- return ret;
- }
-
- /**
- * Applies a reduce transformation on the windowed data stream by reducing
- * the current window at every trigger.The user can also extend the
- * {@link RichReduceFunction} to gain access to other features provided by
- * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
- *
- * @param reduceFunction
- * The reduce function that will be applied to the windows.
- * @return The transformed DataStream
- */
- public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reduceFunction) {
- return dataStream.transform("Window-Reduce", getType(),
- getReduceInvokable(reduceFunction));
- }
-
- /**
- * Applies a reduceGroup transformation on the windowed data stream by
- * reducing the current window at every trigger. In contrast with the
- * standard binary reducer, with reduceGroup the user can access all
- * elements of the window at the same time through the iterable interface.
- * The user can also extend the {@link RichGroupReduceFunction} to gain
- * access to other features provided by the
- * {@link org.apache.flink.api.common.functions.RichFunction} interface.
- *
- * @param reduceFunction
- * The reduce function that will be applied to the windows.
- * @return The transformed DataStream
- */
- public <R> SingleOutputStreamOperator<R, ?> reduceGroup(
- GroupReduceFunction<OUT, R> reduceFunction) {
-
- TypeInformation<OUT> inType = getType();
- TypeInformation<R> outType = TypeExtractor
- .getGroupReduceReturnTypes(reduceFunction, inType);
-
- return dataStream.transform("WindowReduce", outType,
- getReduceGroupInvokable(reduceFunction));
- }
-
- /**
- * Applies a reduceGroup transformation on the windowed data stream by
- * reducing the current window at every trigger. In contrast with the
- * standard binary reducer, with reduceGroup the user can access all
- * elements of the window at the same time through the iterable interface.
- * The user can also extend the {@link RichGroupReduceFunction} to gain
- * access to other features provided by the
- * {@link org.apache.flink.api.common.functions.RichFunction} interface.
- * </br> </br> This version of reduceGroup uses user supplied
- * typeinformation for serializaton. Use this only when the system is unable
- * to detect type information using:
- * {@link #reduceGroup(GroupReduceFunction)}
- *
- * @param reduceFunction
- * The reduce function that will be applied to the windows.
- * @return The transformed DataStream
- */
- public <R> SingleOutputStreamOperator<R, ?> reduceGroup(
- GroupReduceFunction<OUT, R> reduceFunction, TypeInformation<R> outType) {
-
- return dataStream.transform("Window-Reduce", outType,
- getReduceGroupInvokable(reduceFunction));
- }
-
- /**
- * Applies an aggregation that sums every window of the data stream at the
- * given position.
- *
- * @param positionToSum
- * The position in the tuple/array to sum
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> sum(int positionToSum) {
- dataStream.checkFieldRange(positionToSum);
- return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(positionToSum,
- dataStream.getClassAtPos(positionToSum), getType()));
- }
-
- /**
- * Applies an aggregation that sums every window of the pojo data stream at
- * the given field for every window. </br></br> A field expression is either
- * the name of a public field or a getter method with parentheses of the
- * stream's underlying type. A dot can be used to drill down into objects,
- * as in {@code "field1.getInnerField2()" }.
- *
- * @param positionToSum
- * The field to sum
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> sum(String field) {
- return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field, getType()));
- }
-
- /**
- * Applies an aggregation that that gives the minimum value of every window
- * of the data stream at the given position.
- *
- * @param positionToMin
- * The position to minimize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> min(int positionToMin) {
- dataStream.checkFieldRange(positionToMin);
- return aggregate(ComparableAggregator.getAggregator(positionToMin, getType(),
- AggregationType.MIN));
- }
-
- /**
- * Applies an aggregation that that gives the minimum value of the pojo data
- * stream at the given field expression for every window. </br></br>A field
- * expression is either the name of a public field or a getter method with
- * parentheses of the {@link DataStream}S underlying type. A dot can be used
- * to drill down into objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> min(String field) {
- return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MIN,
- false));
- }
-
- /**
- * Applies an aggregation that gives the minimum element of every window of
- * the data stream by the given position. If more elements have the same
- * minimum value the operator returns the first element by default.
- *
- * @param positionToMinBy
- * The position to minimize by
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
- return this.minBy(positionToMinBy, true);
- }
-
- /**
- * Applies an aggregation that gives the minimum element of every window of
- * the data stream by the given position. If more elements have the same
- * minimum value the operator returns the first element by default.
- *
- * @param positionToMinBy
- * The position to minimize by
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(String positionToMinBy) {
- return this.minBy(positionToMinBy, true);
- }
-
- /**
- * Applies an aggregation that gives the minimum element of every window of
- * the data stream by the given position. If more elements have the same
- * minimum value the operator returns either the first or last one depending
- * on the parameter setting.
- *
- * @param positionToMinBy
- * The position to minimize
- * @param first
- * If true, then the operator return the first element with the
- * minimum value, otherwise returns the last
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy, boolean first) {
- dataStream.checkFieldRange(positionToMinBy);
- return aggregate(ComparableAggregator.getAggregator(positionToMinBy, getType(),
- AggregationType.MINBY, first));
- }
-
- /**
- * Applies an aggregation that that gives the minimum element of the pojo
- * data stream by the given field expression for every window. A field
- * expression is either the name of a public field or a getter method with
- * parentheses of the {@link DataStream}S underlying type. A dot can be used
- * to drill down into objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @param first
- * If True then in case of field equality the first object will
- * be returned
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(String field, boolean first) {
- return aggregate(ComparableAggregator.getAggregator(field, getType(),
- AggregationType.MINBY, first));
- }
-
- /**
- * Applies an aggregation that gives the maximum value of every window of
- * the data stream at the given position.
- *
- * @param positionToMax
- * The position to maximize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> max(int positionToMax) {
- dataStream.checkFieldRange(positionToMax);
- return aggregate(ComparableAggregator.getAggregator(positionToMax, getType(),
- AggregationType.MAX));
- }
-
- /**
- * Applies an aggregation that that gives the maximum value of the pojo data
- * stream at the given field expression for every window. A field expression
- * is either the name of a public field or a getter method with parentheses
- * of the {@link DataStream}S underlying type. A dot can be used to drill
- * down into objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> max(String field) {
- return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MAX,
- false));
- }
-
- /**
- * Applies an aggregation that gives the maximum element of every window of
- * the data stream by the given position. If more elements have the same
- * maximum value the operator returns the first by default.
- *
- * @param positionToMaxBy
- * The position to maximize by
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
- return this.maxBy(positionToMaxBy, true);
- }
-
- /**
- * Applies an aggregation that gives the maximum element of every window of
- * the data stream by the given position. If more elements have the same
- * maximum value the operator returns the first by default.
- *
- * @param positionToMaxBy
- * The position to maximize by
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(String positionToMaxBy) {
- return this.maxBy(positionToMaxBy, true);
- }
-
- /**
- * Applies an aggregation that gives the maximum element of every window of
- * the data stream by the given position. If more elements have the same
- * maximum value the operator returns either the first or last one depending
- * on the parameter setting.
- *
- * @param positionToMaxBy
- * The position to maximize by
- * @param first
- * If true, then the operator return the first element with the
- * maximum value, otherwise returns the last
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
- dataStream.checkFieldRange(positionToMaxBy);
- return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getType(),
- AggregationType.MAXBY, first));
- }
-
- /**
- * Applies an aggregation that that gives the maximum element of the pojo
- * data stream by the given field expression for every window. A field
- * expression is either the name of a public field or a getter method with
- * parentheses of the {@link DataStream}S underlying type. A dot can be used
- * to drill down into objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @param first
- * If True then in case of field equality the first object will
- * be returned
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(String field, boolean first) {
- return aggregate(ComparableAggregator.getAggregator(field, getType(),
- AggregationType.MAXBY, first));
- }
-
- private SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregator) {
- StreamInvokable<OUT, OUT> invokable = getReduceInvokable(aggregator);
-
- SingleOutputStreamOperator<OUT, ?> returnStream = dataStream.transform("Window-Aggregation",
- getType(), invokable);
-
- return returnStream;
- }
-
- private LinkedList<TriggerPolicy<OUT>> getTriggers() {
-
- LinkedList<TriggerPolicy<OUT>> triggers = new LinkedList<TriggerPolicy<OUT>>();
-
- if (triggerHelpers != null) {
- for (WindowingHelper<OUT> helper : triggerHelpers) {
- triggers.add(helper.toTrigger());
- }
- }
-
- if (userTriggers != null) {
- triggers.addAll(userTriggers);
- }
-
- return triggers;
-
- }
-
- private LinkedList<EvictionPolicy<OUT>> getEvicters() {
-
- LinkedList<EvictionPolicy<OUT>> evicters = new LinkedList<EvictionPolicy<OUT>>();
-
- if (evictionHelpers != null) {
- for (WindowingHelper<OUT> helper : evictionHelpers) {
- evicters.add(helper.toEvict());
- }
- } else {
- if (userEvicters == null) {
- boolean notOnlyTime=false;
- for (WindowingHelper<OUT> helper : triggerHelpers){
- if (helper instanceof Time<?>){
- evicters.add(helper.toEvict());
- } else {
- notOnlyTime=true;
- }
- }
- if (notOnlyTime){
- evicters.add(new TumblingEvictionPolicy<OUT>());
- }
- }
- }
-
- if (userEvicters != null) {
- evicters.addAll(userEvicters);
- }
-
- return evicters;
- }
-
- private LinkedList<TriggerPolicy<OUT>> getCentralTriggers() {
- LinkedList<TriggerPolicy<OUT>> cTriggers = new LinkedList<TriggerPolicy<OUT>>();
- if (allCentral) {
- cTriggers.addAll(getTriggers());
- } else {
- for (TriggerPolicy<OUT> trigger : getTriggers()) {
- if (trigger instanceof TimeTriggerPolicy) {
- cTriggers.add(trigger);
- }
- }
- }
- return cTriggers;
- }
-
- private LinkedList<CloneableTriggerPolicy<OUT>> getDistributedTriggers() {
- LinkedList<CloneableTriggerPolicy<OUT>> dTriggers = null;
-
- if (!allCentral) {
- dTriggers = new LinkedList<CloneableTriggerPolicy<OUT>>();
- for (TriggerPolicy<OUT> trigger : getTriggers()) {
- if (!(trigger instanceof TimeTriggerPolicy)) {
- dTriggers.add((CloneableTriggerPolicy<OUT>) trigger);
- }
- }
- }
-
- return dTriggers;
- }
-
- private LinkedList<CloneableEvictionPolicy<OUT>> getDistributedEvicters() {
- LinkedList<CloneableEvictionPolicy<OUT>> evicters = null;
-
- if (!allCentral) {
- evicters = new LinkedList<CloneableEvictionPolicy<OUT>>();
- for (EvictionPolicy<OUT> evicter : getEvicters()) {
- evicters.add((CloneableEvictionPolicy<OUT>) evicter);
- }
- }
-
- return evicters;
- }
-
- private LinkedList<EvictionPolicy<OUT>> getCentralEvicters() {
- if (allCentral) {
- return getEvicters();
- } else {
- return null;
- }
- }
-
- private <R> StreamInvokable<OUT, R> getReduceGroupInvokable(GroupReduceFunction<OUT, R> reducer) {
- StreamInvokable<OUT, R> invokable;
- if (isGrouped) {
- invokable = new GroupedWindowInvokable<OUT, R>(clean(reducer), keySelector,
- getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers(),
- getCentralEvicters());
-
- } else {
- invokable = new WindowGroupReduceInvokable<OUT, R>(clean(reducer), getTriggers(),
- getEvicters());
- }
- return invokable;
- }
-
- private StreamInvokable<OUT, OUT> getReduceInvokable(ReduceFunction<OUT> reducer) {
- StreamInvokable<OUT, OUT> invokable;
- if (isGrouped) {
- invokable = new GroupedWindowInvokable<OUT, OUT>(clean(reducer), keySelector,
- getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers(),
- getCentralEvicters());
-
- } else {
- invokable = new WindowReduceInvokable<OUT>(clean(reducer), getTriggers(), getEvicters());
- }
- return invokable;
- }
-
- /**
- * Gets the output type.
- *
- * @return The output type.
- */
- public TypeInformation<OUT> getType() {
- return dataStream.getType();
- }
-
- public DataStream<OUT> getDataStream() {
- return dataStream;
- }
-
- protected WindowedDataStream<OUT> copy() {
- return new WindowedDataStream<OUT>(this);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
deleted file mode 100644
index 03160c2..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream.temporaloperator;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.api.common.functions.CrossFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.operators.CrossOperator;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
-import org.apache.flink.streaming.api.function.co.CrossWindowFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
-
-public class StreamCrossOperator<I1, I2> extends
- TemporalOperator<I1, I2, StreamCrossOperator.CrossWindow<I1, I2>> {
-
- public StreamCrossOperator(DataStream<I1> input1, DataStream<I2> input2) {
- super(input1, input2);
- }
-
- @Override
- protected CrossWindow<I1, I2> createNextWindowOperator() {
-
- CrossWindowFunction<I1, I2, Tuple2<I1, I2>> crossWindowFunction = new CrossWindowFunction<I1, I2, Tuple2<I1, I2>>(
- input1.clean(new CrossOperator.DefaultCrossFunction<I1, I2>()));
-
- return new CrossWindow<I1, I2>(this, input1.connect(input2).addGeneralWindowCombine(
- crossWindowFunction,
- new TupleTypeInfo<Tuple2<I1, I2>>(input1.getType(), input2.getType()), windowSize,
- slideInterval, timeStamp1, timeStamp2));
- }
-
- public static class CrossWindow<I1, I2> extends
- SingleOutputStreamOperator<Tuple2<I1, I2>, CrossWindow<I1, I2>> implements
- TemporalWindow<CrossWindow<I1, I2>> {
-
- private StreamCrossOperator<I1, I2> op;
-
- public CrossWindow(StreamCrossOperator<I1, I2> op, DataStream<Tuple2<I1, I2>> ds) {
- super(ds);
- this.op = op;
- }
-
- public CrossWindow<I1, I2> every(long length, TimeUnit timeUnit) {
- return every(timeUnit.toMillis(length));
- }
-
- @SuppressWarnings("unchecked")
- public CrossWindow<I1, I2> every(long length) {
- ((CoWindowInvokable<I1, I2, ?>) streamGraph.getInvokable(id)).setSlideSize(length);
- return this;
- }
-
- /**
- * Finalizes a temporal Cross transformation by applying a
- * {@link CrossFunction} to each pair of crossed elements.<br/>
- * Each CrossFunction call returns exactly one element.
- *
- * @param function
- * The CrossFunction that is called for each pair of crossed
- * elements.
- * @return The crossed data streams
- *
- */
- public <R> SingleOutputStreamOperator<R, ?> with(CrossFunction<I1, I2, R> function) {
- TypeInformation<R> outTypeInfo = TypeExtractor.getCrossReturnTypes(function,
- op.input1.getType(), op.input2.getType());
-
- CoWindowInvokable<I1, I2, R> invokable = new CoWindowInvokable<I1, I2, R>(
- new CrossWindowFunction<I1, I2, R>(clean(function)), op.windowSize,
- op.slideInterval, op.timeStamp1, op.timeStamp2);
-
- streamGraph.setInvokable(id, invokable);
-
- return setType(outTypeInfo);
-
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
deleted file mode 100644
index d2b2032..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream.temporaloperator;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.operators.Keys;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
-import org.apache.flink.streaming.api.function.co.JoinWindowFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
-import org.apache.flink.streaming.util.keys.KeySelectorUtil;
-
-public class StreamJoinOperator<I1, I2> extends
- TemporalOperator<I1, I2, StreamJoinOperator.JoinWindow<I1, I2>> {
-
- public StreamJoinOperator(DataStream<I1> input1, DataStream<I2> input2) {
- super(input1, input2);
- }
-
- @Override
- protected JoinWindow<I1, I2> createNextWindowOperator() {
- return new JoinWindow<I1, I2>(this);
- }
-
- public static class JoinWindow<I1, I2> implements TemporalWindow<JoinWindow<I1, I2>> {
-
- private StreamJoinOperator<I1, I2> op;
- private TypeInformation<I1> type1;
-
- private JoinWindow(StreamJoinOperator<I1, I2> operator) {
- this.op = operator;
- this.type1 = op.input1.getType();
- }
-
- /**
- * Continues a temporal Join transformation. <br/>
- * Defines the {@link Tuple} fields of the first join {@link DataStream}
- * that should be used as join keys.<br/>
- * <b>Note: Fields can only be selected as join keys on Tuple
- * DataStreams.</b><br/>
- *
- * @param fields
- * The indexes of the other Tuple fields of the first join
- * DataStreams that should be used as keys.
- * @return An incomplete Join transformation. Call
- * {@link JoinPredicate#equalTo} to continue the Join.
- */
- public JoinPredicate<I1, I2> where(int... fields) {
- return new JoinPredicate<I1, I2>(op, KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys<I1>(fields, type1), type1));
- }
-
- /**
- * Continues a temporal join transformation. <br/>
- * Defines the fields of the first join {@link DataStream} that should
- * be used as grouping keys. Fields are the names of member fields of
- * the underlying type of the data stream.
- *
- * @param fields
- * The fields of the first join DataStream that should be
- * used as keys.
- * @return An incomplete Join transformation. Call
- * {@link JoinPredicate#equalTo} to continue the Join.
- */
- public JoinPredicate<I1, I2> where(String... fields) {
- return new JoinPredicate<I1, I2>(op, KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys<I1>(fields, type1), type1));
- }
-
- /**
- * Continues a temporal Join transformation and defines a
- * {@link KeySelector} function for the first join {@link DataStream}
- * .</br> The KeySelector function is called for each element of the
- * first DataStream and extracts a single key value on which the
- * DataStream is joined. </br>
- *
- * @param keySelector
- * The KeySelector function which extracts the key values
- * from the DataStream on which it is joined.
- * @return An incomplete Join transformation. Call
- * {@link JoinPredicate#equalTo} to continue the Join.
- */
- public <K> JoinPredicate<I1, I2> where(KeySelector<I1, K> keySelector) {
- return new JoinPredicate<I1, I2>(op, keySelector);
- }
-
- @Override
- public JoinWindow<I1, I2> every(long length, TimeUnit timeUnit) {
- return every(timeUnit.toMillis(length));
- }
-
- @Override
- public JoinWindow<I1, I2> every(long length) {
- op.slideInterval = length;
- return this;
- }
-
- // ----------------------------------------------------------------------------------------
-
- }
-
- /**
- * Intermediate step of a temporal Join transformation. <br/>
- * To continue the Join transformation, select the join key of the second
- * input {@link DataStream} by calling {@link JoinPredicate#equalTo}
- *
- */
- public static class JoinPredicate<I1, I2> {
-
- private StreamJoinOperator<I1, I2> op;
- private KeySelector<I1, ?> keys1;
- private KeySelector<I2, ?> keys2;
- private TypeInformation<I2> type2;
-
- private JoinPredicate(StreamJoinOperator<I1, I2> operator, KeySelector<I1, ?> keys1) {
- this.op = operator;
- this.keys1 = keys1;
- this.type2 = op.input2.getType();
- }
-
- /**
- * Creates a temporal Join transformation and defines the {@link Tuple}
- * fields of the second join {@link DataStream} that should be used as
- * join keys.<br/>
- * </p> The resulting operator wraps each pair of joining elements in a
- * Tuple2<I1,I2>(first, second). To use a different wrapping function
- * use {@link JoinedStream#with(JoinFunction)}
- *
- * @param fields
- * The indexes of the Tuple fields of the second join
- * DataStream that should be used as keys.
- * @return A streaming join operator. Call {@link JoinedStream#with} to
- * apply a custom wrapping
- */
- public JoinedStream<I1, I2> equalTo(int... fields) {
- keys2 = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<I2>(fields, type2),
- type2);
- return createJoinOperator();
- }
-
- /**
- * Creates a temporal Join transformation and defines the fields of the
- * second join {@link DataStream} that should be used as join keys. </p>
- * The resulting operator wraps each pair of joining elements in a
- * Tuple2<I1,I2>(first, second). To use a different wrapping function
- * use {@link JoinedStream#with(JoinFunction)}
- *
- * @param fields
- * The fields of the second join DataStream that should be
- * used as keys.
- * @return A streaming join operator. Call {@link JoinedStream#with} to
- * apply a custom wrapping
- */
- public JoinedStream<I1, I2> equalTo(String... fields) {
- this.keys2 = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<I2>(fields,
- type2), type2);
- return createJoinOperator();
- }
-
- /**
- * Creates a temporal Join transformation and defines a
- * {@link KeySelector} function for the second join {@link DataStream}
- * .</br> The KeySelector function is called for each element of the
- * second DataStream and extracts a single key value on which the
- * DataStream is joined. </p> The resulting operator wraps each pair of
- * joining elements in a Tuple2<I1,I2>(first, second). To use a
- * different wrapping function use
- * {@link JoinedStream#with(JoinFunction)}
- *
- *
- * @param keySelector
- * The KeySelector function which extracts the key values
- * from the second DataStream on which it is joined.
- * @return A streaming join operator. Call {@link JoinedStream#with} to
- * apply a custom wrapping
- */
- public <K> JoinedStream<I1, I2> equalTo(KeySelector<I2, K> keySelector) {
- this.keys2 = keySelector;
- return createJoinOperator();
- }
-
- private JoinedStream<I1, I2> createJoinOperator() {
-
- JoinFunction<I1, I2, Tuple2<I1, I2>> joinFunction = new DefaultJoinFunction<I1, I2>();
-
- JoinWindowFunction<I1, I2, Tuple2<I1, I2>> joinWindowFunction = getJoinWindowFunction(
- joinFunction, this);
-
- TypeInformation<Tuple2<I1, I2>> outType = new TupleTypeInfo<Tuple2<I1, I2>>(
- op.input1.getType(), op.input2.getType());
-
- return new JoinedStream<I1, I2>(this, op.input1
- .groupBy(keys1)
- .connect(op.input2.groupBy(keys2))
- .addGeneralWindowCombine(joinWindowFunction, outType, op.windowSize,
- op.slideInterval, op.timeStamp1, op.timeStamp2));
- }
- }
-
- public static class JoinedStream<I1, I2> extends
- SingleOutputStreamOperator<Tuple2<I1, I2>, JoinedStream<I1, I2>> {
- private final JoinPredicate<I1, I2> predicate;
-
- private JoinedStream(JoinPredicate<I1, I2> predicate, DataStream<Tuple2<I1, I2>> ds) {
- super(ds);
- this.predicate = predicate;
- }
-
- /**
- * Completes a stream join. </p> The resulting operator wraps each pair
- * of joining elements using the user defined {@link JoinFunction}
- *
- * @return The joined data stream.
- */
- public <OUT> SingleOutputStreamOperator<OUT, ?> with(JoinFunction<I1, I2, OUT> joinFunction) {
-
- TypeInformation<OUT> outType = TypeExtractor.getJoinReturnTypes(joinFunction,
- predicate.op.input1.getType(), predicate.op.input2.getType());
-
- CoWindowInvokable<I1, I2, OUT> invokable = new CoWindowInvokable<I1, I2, OUT>(
- getJoinWindowFunction(joinFunction, predicate), predicate.op.windowSize,
- predicate.op.slideInterval, predicate.op.timeStamp1, predicate.op.timeStamp2);
-
- streamGraph.setInvokable(id, invokable);
-
- return setType(outType);
- }
- }
-
- public static final class DefaultJoinFunction<I1, I2> implements
- JoinFunction<I1, I2, Tuple2<I1, I2>> {
-
- private static final long serialVersionUID = 1L;
- private final Tuple2<I1, I2> outTuple = new Tuple2<I1, I2>();
-
- @Override
- public Tuple2<I1, I2> join(I1 first, I2 second) throws Exception {
- outTuple.f0 = first;
- outTuple.f1 = second;
- return outTuple;
- }
- }
-
- public static <I1, I2, OUT> JoinWindowFunction<I1, I2, OUT> getJoinWindowFunction(
- JoinFunction<I1, I2, OUT> joinFunction, JoinPredicate<I1, I2> predicate) {
- return new JoinWindowFunction<I1, I2, OUT>(predicate.keys1, predicate.keys2, joinFunction);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java
deleted file mode 100644
index f121dfa..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream.temporaloperator;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.windowing.helper.SystemTimestamp;
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-
-public abstract class TemporalOperator<I1, I2, OP extends TemporalWindow<OP>> {
-
- public final DataStream<I1> input1;
- public final DataStream<I2> input2;
-
- public long windowSize;
- public long slideInterval;
-
- public TimestampWrapper<I1> timeStamp1;
- public TimestampWrapper<I2> timeStamp2;
-
- public TemporalOperator(DataStream<I1> input1, DataStream<I2> input2) {
- if (input1 == null || input2 == null) {
- throw new NullPointerException();
- }
- this.input1 = input1.copy();
- this.input2 = input2.copy();
- }
-
- /**
- * Continues a temporal transformation.<br/>
- * Defines the window size on which the two DataStreams will be transformed.
- * To define sliding windows call {@link TemporalWindow#every} on the
- * resulting operator.
- *
- * @param length
- * The size of the window in milliseconds.
- * @param timeUnit
- * The unit if time to be used
- * @return An incomplete temporal transformation.
- */
- @SuppressWarnings("unchecked")
- public OP onWindow(long length, TimeUnit timeUnit) {
- return onWindow(timeUnit.toMillis(length),
- (TimestampWrapper<I1>) SystemTimestamp.getWrapper(),
- (TimestampWrapper<I2>) SystemTimestamp.getWrapper());
- }
-
- /**
- * Continues a temporal transformation.<br/>
- * Defines the window size on which the two DataStreams will be
- * transformed.To define sliding windows call {@link TemporalWindow#every}
- * on the resulting operator.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @param timeStamp1
- * The timestamp used to extract time from the elements of the
- * first data stream.
- * @param timeStamp2
- * The timestamp used to extract time from the elements of the
- * second data stream.
- * @return An incomplete temporal transformation.
- */
- public OP onWindow(long length, Timestamp<I1> timeStamp1, Timestamp<I2> timeStamp2) {
- return onWindow(length, timeStamp1, timeStamp2, 0);
- }
-
- /**
- * Continues a temporal transformation.<br/>
- * Defines the window size on which the two DataStreams will be
- * transformed.To define sliding windows call {@link TemporalWindow#every}
- * on the resulting operator.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @param timeStamp1
- * The timestamp used to extract time from the elements of the
- * first data stream.
- * @param timeStamp2
- * The timestamp used to extract time from the elements of the
- * second data stream.
- * @param startTime
- * The start time to measure the first window
- * @return An incomplete temporal transformation.
- */
- public OP onWindow(long length, Timestamp<I1> timeStamp1, Timestamp<I2> timeStamp2,
- long startTime) {
- return onWindow(length, new TimestampWrapper<I1>(timeStamp1, startTime),
- new TimestampWrapper<I2>(timeStamp2, startTime));
- }
-
- private OP onWindow(long length, TimestampWrapper<I1> timeStamp1,
- TimestampWrapper<I2> timeStamp2) {
-
- this.windowSize = length;
- this.slideInterval = length;
-
- this.timeStamp1 = timeStamp1;
- this.timeStamp2 = timeStamp2;
-
- return createNextWindowOperator();
- }
-
- protected abstract OP createNextWindowOperator();
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java
deleted file mode 100644
index 8ac1492..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream.temporaloperator;
-
-import java.util.concurrent.TimeUnit;
-
-public interface TemporalWindow<T> {
-
- /**
- * Defines the slide interval for this temporal operator
- *
- * @param length
- * Length of the window
- * @param timeUnit
- * Unit of time
- * @return The temporal operator with slide interval specified
- */
- public T every(long length, TimeUnit timeUnit);
-
- /**
- * Defines the slide interval for this temporal operator
- *
- * @param length
- * Length of the window
- * @return The temporal operator with slide interval specified
- */
- public T every(long length);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
deleted file mode 100755
index 4824fca..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.environment;
-
-import org.apache.flink.streaming.util.ClusterUtil;
-
-public class LocalStreamEnvironment extends StreamExecutionEnvironment {
-
- protected static ClassLoader userClassLoader;
-
- /**
- * Executes the JobGraph of the on a mini cluster of CLusterUtil with a
- * default name.
- */
- @Override
- public void execute() throws Exception {
- ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(), getDegreeOfParallelism());
- }
-
- /**
- * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
- * specified name.
- *
- * @param jobName
- * name of the job
- */
- @Override
- public void execute(String jobName) throws Exception {
- ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(jobName),
- getDegreeOfParallelism());
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
deleted file mode 100644
index 2eb05ad..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.environment;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.client.program.Client;
-import org.apache.flink.client.program.JobWithJars;
-import org.apache.flink.client.program.ProgramInvocationException;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class RemoteStreamEnvironment extends StreamExecutionEnvironment {
- private static final Logger LOG = LoggerFactory.getLogger(RemoteStreamEnvironment.class);
-
- private String host;
- private int port;
- private List<File> jarFiles;
-
- /**
- * Creates a new RemoteStreamEnvironment that points to the master
- * (JobManager) described by the given host name and port.
- *
- * @param host
- * The host name or address of the master (JobManager), where the
- * program should be executed.
- * @param port
- * The port of the master (JobManager), where the program should
- * be executed.
- * @param jarFiles
- * The JAR files with code that needs to be shipped to the
- * cluster. If the program uses user-defined functions,
- * user-defined input formats, or any libraries, those must be
- * provided in the JAR files.
- */
- public RemoteStreamEnvironment(String host, int port, String... jarFiles) {
- if (host == null) {
- throw new NullPointerException("Host must not be null.");
- }
-
- if (port < 1 || port >= 0xffff) {
- throw new IllegalArgumentException("Port out of range");
- }
-
- this.host = host;
- this.port = port;
- this.jarFiles = new ArrayList<File>();
- for (int i = 0; i < jarFiles.length; i++) {
- File file = new File(jarFiles[i]);
- try {
- JobWithJars.checkJarFile(file);
- } catch (IOException e) {
- throw new RuntimeException("Problem with jar file " + file.getAbsolutePath(), e);
- }
- this.jarFiles.add(file);
- }
- }
-
- @Override
- public void execute() {
-
- JobGraph jobGraph = streamGraph.getJobGraph();
- executeRemotely(jobGraph);
- }
-
- @Override
- public void execute(String jobName) {
-
- JobGraph jobGraph = streamGraph.getJobGraph(jobName);
- executeRemotely(jobGraph);
- }
-
- /**
- * Executes the remote job.
- *
- * @param jobGraph
- * jobGraph to execute
- */
- private void executeRemotely(JobGraph jobGraph) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Running remotely at {}:{}", host, port);
- }
-
- for (File file : jarFiles) {
- jobGraph.addJar(new Path(file.getAbsolutePath()));
- }
-
- Configuration configuration = jobGraph.getJobConfiguration();
- Client client = new Client(new InetSocketAddress(host, port), configuration,
- JobWithJars.buildUserCodeClassLoader(jarFiles, JobWithJars.class.getClassLoader()));
-
- try {
- client.run(jobGraph, true);
- } catch (ProgramInvocationException e) {
- throw new RuntimeException("Cannot execute job due to ProgramInvocationException", e);
- }
- }
-
- @Override
- public String toString() {
- return "Remote Environment (" + this.host + ":" + this.port + " - DOP = "
- + (getDegreeOfParallelism() == -1 ? "default" : getDegreeOfParallelism()) + ")";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
deleted file mode 100644
index 7d41d2a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.environment;
-
-import java.io.File;
-import java.util.List;
-
-import org.apache.flink.client.program.Client;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-
-public class StreamContextEnvironment extends StreamExecutionEnvironment {
-
- protected static ClassLoader userClassLoader;
- protected List<File> jars;
- protected Client client;
-
- protected StreamContextEnvironment(Client client, List<File> jars, int dop) {
- this.client = client;
- this.jars = jars;
- if (dop > 0) {
- setDegreeOfParallelism(dop);
- } else {
- setDegreeOfParallelism(GlobalConfiguration.getInteger(
- ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE_KEY,
- ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE));
- }
- }
-
- @Override
- public void execute() throws Exception {
- execute(null);
- }
-
- @Override
- public void execute(String jobName) throws Exception {
- currentEnvironment = null;
-
- JobGraph jobGraph;
- if (jobName == null) {
- jobGraph = this.streamGraph.getJobGraph();
- } else {
- jobGraph = this.streamGraph.getJobGraph(jobName);
- }
-
- for (File file : jars) {
- jobGraph.addJar(new Path(file.getAbsolutePath()));
- }
-
- try {
- client.run(jobGraph, true);
-
- } catch (Exception e) {
- throw e;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
deleted file mode 100644
index 45c14c1..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ /dev/null
@@ -1,618 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.environment;
-
-import java.io.File;
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.Validate;
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.TextInputFormat;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.client.program.Client;
-import org.apache.flink.client.program.Client.OptimizerPlanEnvironment;
-import org.apache.flink.client.program.ContextEnvironment;
-import org.apache.flink.client.program.PackagedProgram.PreviewPlanEnvironment;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.streaming.api.StreamGraph;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
-import org.apache.flink.streaming.api.function.source.FileMonitoringFunction;
-import org.apache.flink.streaming.api.function.source.FileMonitoringFunction.WatchType;
-import org.apache.flink.streaming.api.function.source.FileReadFunction;
-import org.apache.flink.streaming.api.function.source.FileSourceFunction;
-import org.apache.flink.streaming.api.function.source.FromElementsFunction;
-import org.apache.flink.streaming.api.function.source.GenSequenceFunction;
-import org.apache.flink.streaming.api.function.source.GenericSourceFunction;
-import org.apache.flink.streaming.api.function.source.ParallelSourceFunction;
-import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.api.function.source.SocketTextStreamFunction;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.invokable.SourceInvokable;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-
-/**
- * {@link ExecutionEnvironment} for streaming jobs. An instance of it is
- * necessary to construct streaming topologies.
- *
- */
-public abstract class StreamExecutionEnvironment {
-
- private static int defaultLocalDop = Runtime.getRuntime().availableProcessors();
-
- private long bufferTimeout = 100;
-
- private ExecutionConfig config = new ExecutionConfig();
-
- protected static StreamExecutionEnvironment currentEnvironment;
-
- protected StreamGraph streamGraph;
-
- // --------------------------------------------------------------------------------------------
- // Constructor and Properties
- // --------------------------------------------------------------------------------------------
-
- /**
- * Constructor for creating StreamExecutionEnvironment
- */
- protected StreamExecutionEnvironment() {
- streamGraph = new StreamGraph();
- }
-
- /**
- * Sets the config object.
- */
- public void setConfig(ExecutionConfig config) {
- Validate.notNull(config);
- this.config = config;
- }
-
- /**
- * Gets the config object.
- */
- public ExecutionConfig getConfig() {
- return config;
- }
-
- /**
- * Gets the degree of parallelism with which operation are executed by
- * default. Operations can individually override this value to use a
- * specific degree of parallelism.
- *
- * @return The degree of parallelism used by operations, unless they
- * override that value.
- */
- public int getDegreeOfParallelism() {
- return config.getDegreeOfParallelism();
- }
-
- /**
- * Sets the degree of parallelism (DOP) for operations executed through this
- * environment. Setting a DOP of x here will cause all operators (such as
- * map, batchReduce) to run with x parallel instances. This method overrides
- * the default parallelism for this environment. The
- * {@link LocalStreamEnvironment} uses by default a value equal to the
- * number of hardware contexts (CPU cores / threads). When executing the
- * program via the command line client from a JAR file, the default degree
- * of parallelism is the one configured for that setup.
- *
- * @param degreeOfParallelism
- * The degree of parallelism
- */
- public StreamExecutionEnvironment setDegreeOfParallelism(int degreeOfParallelism) {
- if (degreeOfParallelism < 1) {
- throw new IllegalArgumentException("Degree of parallelism must be at least one.");
- }
- config.setDegreeOfParallelism(degreeOfParallelism);
- return this;
- }
-
- /**
- * Sets the maximum time frequency (milliseconds) for the flushing of the
- * output buffers. By default the output buffers flush frequently to provide
- * low latency and to aid smooth developer experience. Setting the parameter
- * can result in three logical modes:
- *
- * <ul>
- * <li>
- * A positive integer triggers flushing periodically by that integer</li>
- * <li>
- * 0 triggers flushing after every record thus minimizing latency</li>
- * <li>
- * -1 triggers flushing only when the output buffer is full thus maximizing
- * throughput</li>
- * </ul>
- *
- * @param timeoutMillis
- * The maximum time between two output flushes.
- */
- public StreamExecutionEnvironment setBufferTimeout(long timeoutMillis) {
- if (timeoutMillis < -1) {
- throw new IllegalArgumentException("Timeout of buffer must be non-negative or -1");
- }
-
- this.bufferTimeout = timeoutMillis;
- return this;
- }
-
- /**
- * Sets the maximum time frequency (milliseconds) for the flushing of the
- * output buffers. For clarification on the extremal values see
- * {@link #setBufferTimeout(long)}.
- *
- * @return The timeout of the buffer.
- */
- public long getBufferTimeout() {
- return this.bufferTimeout;
- }
-
- /**
- * Sets the default parallelism that will be used for the local execution
- * environment created by {@link #createLocalEnvironment()}.
- *
- * @param degreeOfParallelism
- * The degree of parallelism to use as the default local
- * parallelism.
- */
- public static void setDefaultLocalParallelism(int degreeOfParallelism) {
- defaultLocalDop = degreeOfParallelism;
- }
-
- // --------------------------------------------------------------------------------------------
- // Data stream creations
- // --------------------------------------------------------------------------------------------
-
- /**
- * Creates a DataStream that represents the Strings produced by reading the
- * given file line wise. The file will be read with the system's default
- * character set.
- *
- * @param filePath
- * The path of the file, as a URI (e.g.,
- * "file:///some/local/file" or "hdfs://host:port/file/path").
- * @return The DataStream representing the text file.
- */
- public DataStreamSource<String> readTextFile(String filePath) {
- Validate.notNull(filePath, "The file path may not be null.");
- TextInputFormat format = new TextInputFormat(new Path(filePath));
- TypeInformation<String> typeInfo = BasicTypeInfo.STRING_TYPE_INFO;
-
- return addFileSource(format, typeInfo);
- }
-
- /**
- * Creates a DataStream that represents the Strings produced by reading the
- * given file line wise. The file will be read with the given character set.
- *
- * @param filePath
- * The path of the file, as a URI (e.g.,
- * "file:///some/local/file" or "hdfs://host:port/file/path").
- * @return The DataStream representing the text file.
- */
- public DataStreamSource<String> readTextFile(String filePath, String charsetName) {
- Validate.notNull(filePath, "The file path may not be null.");
- TextInputFormat format = new TextInputFormat(new Path(filePath));
- TypeInformation<String> typeInfo = BasicTypeInfo.STRING_TYPE_INFO;
- format.setCharsetName(charsetName);
-
- return addFileSource(format, typeInfo);
- }
-
- /**
- * Creates a DataStream that contains the contents of file created while
- * system watches the given path. The file will be read with the system's
- * default character set.
- *
- * @param filePath
- * The path of the file, as a URI (e.g.,
- * "file:///some/local/file" or "hdfs://host:port/file/path/").
- * @param intervalMillis
- * The interval of file watching in milliseconds.
- * @param watchType
- * The watch type of file stream. When watchType is
- * {@link WatchType.ONLY_NEW_FILES}, the system processes only
- * new files. {@link WatchType.REPROCESS_WITH_APPENDED} means
- * that the system re-processes all contents of appended file.
- * {@link WatchType.PROCESS_ONLY_APPENDED} means that the system
- * processes only appended contents of files.
- *
- * @return The DataStream containing the given directory.
- */
- public DataStream<String> readFileStream(String filePath, long intervalMillis,
- WatchType watchType) {
- DataStream<Tuple3<String, Long, Long>> source = addSource(new FileMonitoringFunction(
- filePath, intervalMillis, watchType), null, "File Stream");
-
- return source.flatMap(new FileReadFunction());
- }
-
- /**
- * Creates a new DataStream that contains the given elements. The elements
- * must all be of the same type, for example, all of the String or Integer.
- * The sequence of elements must not be empty. Furthermore, the elements
- * must be serializable (as defined in java.io.Serializable), because the
- * execution environment may ship the elements into the cluster.
- *
- * @param data
- * The collection of elements to create the DataStream from.
- * @param <OUT>
- * type of the returned stream
- * @return The DataStream representing the elements.
- */
- public <OUT extends Serializable> DataStreamSource<OUT> fromElements(OUT... data) {
- if (data.length == 0) {
- throw new IllegalArgumentException(
- "fromElements needs at least one element as argument");
- }
-
- TypeInformation<OUT> outTypeInfo = TypeExtractor.getForObject(data[0]);
-
- SourceFunction<OUT> function = new FromElementsFunction<OUT>(data);
-
- return addSource(function, outTypeInfo, "Elements source");
- }
-
- /**
- * Creates a DataStream from the given non-empty collection. The type of the
- * DataStream is that of the elements in the collection. The elements need
- * to be serializable (as defined by java.io.Serializable), because the
- * framework may move the elements into the cluster if needed.
- *
- * @param data
- * The collection of elements to create the DataStream from.
- * @param <OUT>
- * type of the returned stream
- * @return The DataStream representing the elements.
- */
- public <OUT extends Serializable> DataStreamSource<OUT> fromCollection(Collection<OUT> data) {
- if (data == null) {
- throw new NullPointerException("Collection must not be null");
- }
-
- if (data.isEmpty()) {
- throw new IllegalArgumentException("Collection must not be empty");
- }
-
- TypeInformation<OUT> outTypeInfo = TypeExtractor.getForObject(data.iterator().next());
- SourceFunction<OUT> function = new FromElementsFunction<OUT>(data);
-
- return addSource(function, outTypeInfo, "Collection Source");
- }
-
- /**
- * Creates a new DataStream that contains the strings received infinitely
- * from socket. Received strings are decoded by the system's default
- * character set.
- *
- * @param hostname
- * The host name which a server socket bind.
- * @param port
- * The port number which a server socket bind. A port number of 0
- * means that the port number is automatically allocated.
- * @param delimiter
- * A character which split received strings into records.
- * @return A DataStream, containing the strings received from socket.
- */
- public DataStreamSource<String> socketTextStream(String hostname, int port, char delimiter) {
- return addSource(new SocketTextStreamFunction(hostname, port, delimiter), null,
- "Socket Stream");
- }
-
- /**
- * Creates a new DataStream that contains the strings received infinitely
- * from socket. Received strings are decoded by the system's default
- * character set, uses '\n' as delimiter.
- *
- * @param hostname
- * The host name which a server socket bind.
- * @param port
- * The port number which a server socket bind. A port number of 0
- * means that the port number is automatically allocated.
- * @return A DataStream, containing the strings received from socket.
- */
- public DataStreamSource<String> socketTextStream(String hostname, int port) {
- return socketTextStream(hostname, port, '\n');
- }
-
- /**
- * Creates a new DataStream that contains a sequence of numbers.
- *
- * @param from
- * The number to start at (inclusive).
- * @param to
- * The number to stop at (inclusive)
- * @return A DataStrean, containing all number in the [from, to] interval.
- */
- public DataStreamSource<Long> generateSequence(long from, long to) {
- if (from > to) {
- throw new IllegalArgumentException("Start of sequence must not be greater than the end");
- }
- return addSource(new GenSequenceFunction(from, to), null, "Sequence Source");
- }
-
- private DataStreamSource<String> addFileSource(InputFormat<String, ?> inputFormat,
- TypeInformation<String> typeInfo) {
- FileSourceFunction function = new FileSourceFunction(inputFormat, typeInfo);
- DataStreamSource<String> returnStream = addSource(function, null, "File Source");
- streamGraph.setInputFormat(returnStream.getId(), inputFormat);
- return returnStream;
- }
-
- /**
- * Create a DataStream using a user defined source function for arbitrary
- * source functionality.</p> By default sources have a parallelism of 1. To
- * enable parallel execution, the user defined source should implement
- * {@link ParallelSourceFunction} or extend
- * {@link RichParallelSourceFunction}. In these cases the resulting source
- * will have the parallelism of the environment. To change this afterwards
- * call {@link DataStreamSource#setParallelism(int)}
- *
- *
- * @param function
- * the user defined function
- * @param <OUT>
- * type of the returned stream
- * @return the data stream constructed
- */
- public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function) {
- return addSource(function, null);
- }
-
- /**
- * Ads a data source with a custom type information thus opening a
- * {@link DataStream}. Only in very special cases does the user need to
- * support type information. Otherwise use
- * {@link #addSource(SourceFunction)} </p> By default sources have a
- * parallelism of 1. To enable parallel execution, the user defined source
- * should implement {@link ParallelSourceFunction} or extend
- * {@link RichParallelSourceFunction}. In these cases the resulting source
- * will have the parallelism of the environment. To change this afterwards
- * call {@link DataStreamSource#setParallelism(int)}
- *
- * @param function
- * the user defined function
- * @param outTypeInfo
- * the user defined type information for the stream
- * @param <OUT>
- * type of the returned stream
- * @return the data stream constructed
- */
- public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function,
- TypeInformation<OUT> outTypeInfo) {
- return addSource(function, outTypeInfo, "Custom Source");
- }
-
- /**
- * Ads a data source with a custom type information thus opening a
- * {@link DataStream}. Only in very special cases does the user need to
- * support type information. Otherwise use
- * {@link #addSource(SourceFunction)}
- *
- * @param function
- * the user defined function
- * @param outTypeInfo
- * the user defined type information for the stream
- * @param sourceName
- * Name of the data source
- * @param <OUT>
- * type of the returned stream
- * @return the data stream constructed
- */
- @SuppressWarnings("unchecked")
- private <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function,
- TypeInformation<OUT> outTypeInfo, String sourceName) {
-
- if (outTypeInfo == null) {
- if (function instanceof GenericSourceFunction) {
- outTypeInfo = ((GenericSourceFunction<OUT>) function).getType();
- } else {
- outTypeInfo = TypeExtractor.createTypeInfo(SourceFunction.class,
- function.getClass(), 0, null, null);
- }
- }
-
- boolean isParallel = function instanceof ParallelSourceFunction;
- int dop = isParallel ? getDegreeOfParallelism() : 1;
-
- StreamInvokable<OUT, OUT> sourceInvokable = new SourceInvokable<OUT>(function);
-
- DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, sourceName,
- outTypeInfo, sourceInvokable, isParallel);
-
- streamGraph.addSourceVertex(returnStream.getId(), sourceInvokable, null, outTypeInfo,
- sourceName, dop);
-
- return returnStream;
- }
-
- // --------------------------------------------------------------------------------------------
- // Instantiation of Execution Contexts
- // --------------------------------------------------------------------------------------------
-
- /**
- * Creates an execution environment that represents the context in which the
- * program is currently executed. If the program is invoked standalone, this
- * method returns a local execution environment, as returned by
- * {@link #createLocalEnvironment()}.
- *
- * @return The execution environment of the context in which the program is
- * executed.
- */
- public static StreamExecutionEnvironment getExecutionEnvironment() {
- if (currentEnvironment != null) {
- return currentEnvironment;
- }
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- if (env instanceof ContextEnvironment) {
- ContextEnvironment ctx = (ContextEnvironment) env;
- currentEnvironment = createContextEnvironment(ctx.getClient(), ctx.getJars(),
- ctx.getDegreeOfParallelism());
- } else if (env instanceof OptimizerPlanEnvironment | env instanceof PreviewPlanEnvironment) {
- currentEnvironment = new StreamPlanEnvironment(env);
- } else {
- return createLocalEnvironment();
- }
- return currentEnvironment;
- }
-
- private static StreamExecutionEnvironment createContextEnvironment(Client client,
- List<File> jars, int dop) {
- return new StreamContextEnvironment(client, jars, dop);
- }
-
- /**
- * Creates a {@link LocalStreamEnvironment}. The local execution environment
- * will run the program in a multi-threaded fashion in the same JVM as the
- * environment was created in. The default degree of parallelism of the
- * local environment is the number of hardware contexts (CPU cores /
- * threads), unless it was specified differently by
- * {@link #setDegreeOfParallelism(int)}.
- *
- * @return A local execution environment.
- */
- public static LocalStreamEnvironment createLocalEnvironment() {
- return createLocalEnvironment(defaultLocalDop);
- }
-
- /**
- * Creates a {@link LocalStreamEnvironment}. The local execution environment
- * will run the program in a multi-threaded fashion in the same JVM as the
- * environment was created in. It will use the degree of parallelism
- * specified in the parameter.
- *
- * @param degreeOfParallelism
- * The degree of parallelism for the local environment.
- * @return A local execution environment with the specified degree of
- * parallelism.
- */
- public static LocalStreamEnvironment createLocalEnvironment(int degreeOfParallelism) {
- currentEnvironment = new LocalStreamEnvironment();
- currentEnvironment.setDegreeOfParallelism(degreeOfParallelism);
- return (LocalStreamEnvironment) currentEnvironment;
- }
-
- // TODO:fix cluster default parallelism
- /**
- * Creates a {@link RemoteStreamEnvironment}. The remote environment sends
- * (parts of) the program to a cluster for execution. Note that all file
- * paths used in the program must be accessible from the cluster. The
- * execution will use no parallelism, unless the parallelism is set
- * explicitly via {@link #setDegreeOfParallelism}.
- *
- * @param host
- * The host name or address of the master (JobManager), where the
- * program should be executed.
- * @param port
- * The port of the master (JobManager), where the program should
- * be executed.
- * @param jarFiles
- * The JAR files with code that needs to be shipped to the
- * cluster. If the program uses user-defined functions,
- * user-defined input formats, or any libraries, those must be
- * provided in the JAR files.
- * @return A remote environment that executes the program on a cluster.
- */
- public static StreamExecutionEnvironment createRemoteEnvironment(String host, int port,
- String... jarFiles) {
- currentEnvironment = new RemoteStreamEnvironment(host, port, jarFiles);
- return currentEnvironment;
- }
-
- /**
- * Creates a {@link RemoteStreamEnvironment}. The remote environment sends
- * (parts of) the program to a cluster for execution. Note that all file
- * paths used in the program must be accessible from the cluster. The
- * execution will use the specified degree of parallelism.
- *
- * @param host
- * The host name or address of the master (JobManager), where the
- * program should be executed.
- * @param port
- * The port of the master (JobManager), where the program should
- * be executed.
- * @param degreeOfParallelism
- * The degree of parallelism to use during the execution.
- * @param jarFiles
- * The JAR files with code that needs to be shipped to the
- * cluster. If the program uses user-defined functions,
- * user-defined input formats, or any libraries, those must be
- * provided in the JAR files.
- * @return A remote environment that executes the program on a cluster.
- */
- public static StreamExecutionEnvironment createRemoteEnvironment(String host, int port,
- int degreeOfParallelism, String... jarFiles) {
- currentEnvironment = new RemoteStreamEnvironment(host, port, jarFiles);
- currentEnvironment.setDegreeOfParallelism(degreeOfParallelism);
- return currentEnvironment;
- }
-
- /**
- * Triggers the program execution. The environment will execute all parts of
- * the program that have resulted in a "sink" operation. Sink operations are
- * for example printing results or forwarding them to a message queue.
- * <p>
- * The program execution will be logged and displayed with a generated
- * default name.
- *
- * @throws Exception
- **/
- public abstract void execute() throws Exception;
-
- /**
- * Triggers the program execution. The environment will execute all parts of
- * the program that have resulted in a "sink" operation. Sink operations are
- * for example printing results or forwarding them to a message queue.
- * <p>
- * The program execution will be logged and displayed with the provided name
- *
- * @param jobName
- * Desired name of the job
- *
- * @throws Exception
- **/
- public abstract void execute(String jobName) throws Exception;
-
- /**
- * Getter of the {@link StreamGraph} of the streaming job.
- *
- * @return The streamgraph representing the transformations
- */
- public StreamGraph getStreamGraph() {
- return streamGraph;
- }
-
- /**
- * Creates the plan with which the system will execute the program, and
- * returns it as a String using a JSON representation of the execution data
- * flow graph. Note that this needs to be called, before the plan is
- * executed.
- *
- * @return The execution plan of the program, as a JSON String.
- */
- public String getExecutionPlan() {
- return getStreamGraph().getStreamingPlanAsJSON();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java
deleted file mode 100644
index 1cff7e7..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.environment;
-
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.client.program.Client;
-import org.apache.flink.client.program.Client.OptimizerPlanEnvironment;
-import org.apache.flink.client.program.PackagedProgram.PreviewPlanEnvironment;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.GlobalConfiguration;
-
-public class StreamPlanEnvironment extends StreamExecutionEnvironment {
-
- private ExecutionEnvironment env;
-
- protected StreamPlanEnvironment(ExecutionEnvironment env) {
- super();
- this.env = env;
-
- int dop = env.getDegreeOfParallelism();
- if (dop > 0) {
- setDegreeOfParallelism(dop);
- } else {
- setDegreeOfParallelism(GlobalConfiguration.getInteger(
- ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE_KEY,
- ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE));
- }
- }
-
- @Override
- public void execute() throws Exception {
- execute("");
- }
-
- @Override
- public void execute(String jobName) throws Exception {
- currentEnvironment = null;
-
- streamGraph.setJobName(jobName);
-
- if (env instanceof OptimizerPlanEnvironment) {
- ((OptimizerPlanEnvironment) env).setPlan(streamGraph);
- } else if (env instanceof PreviewPlanEnvironment) {
- ((PreviewPlanEnvironment) env).setPreview(streamGraph.getStreamingPlanAsJSON());
- }
-
- throw new Client.ProgramAbortException();
- }
-}
[40/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
deleted file mode 100644
index 915ca30..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
+++ /dev/null
@@ -1,1471 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple10;
-import org.apache.flink.api.java.tuple.Tuple11;
-import org.apache.flink.api.java.tuple.Tuple12;
-import org.apache.flink.api.java.tuple.Tuple13;
-import org.apache.flink.api.java.tuple.Tuple14;
-import org.apache.flink.api.java.tuple.Tuple15;
-import org.apache.flink.api.java.tuple.Tuple16;
-import org.apache.flink.api.java.tuple.Tuple17;
-import org.apache.flink.api.java.tuple.Tuple18;
-import org.apache.flink.api.java.tuple.Tuple19;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple20;
-import org.apache.flink.api.java.tuple.Tuple21;
-import org.apache.flink.api.java.tuple.Tuple22;
-import org.apache.flink.api.java.tuple.Tuple23;
-import org.apache.flink.api.java.tuple.Tuple24;
-import org.apache.flink.api.java.tuple.Tuple25;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.api.java.tuple.Tuple6;
-import org.apache.flink.api.java.tuple.Tuple7;
-import org.apache.flink.api.java.tuple.Tuple8;
-import org.apache.flink.api.java.tuple.Tuple9;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.streaming.api.invokable.operator.ProjectInvokable;
-
-public class StreamProjection<IN> {
-
- private DataStream<IN> dataStream;
- private int[] fieldIndexes;
- private TypeInformation<IN> inTypeInfo;
-
- protected StreamProjection(DataStream<IN> dataStream, int[] fieldIndexes) {
- this.dataStream = dataStream;
- this.fieldIndexes = fieldIndexes;
- this.inTypeInfo = dataStream.getType();
- if (!inTypeInfo.isTupleType()) {
- throw new RuntimeException("Only Tuple DataStreams can be projected");
- }
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0> SingleOutputStreamOperator<Tuple1<T0>, ?> types(Class<T0> type0) {
- Class<?>[] types = { type0 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple1<T0>> outType = (TypeInformation<Tuple1<T0>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream.transform("Projection", outType, new ProjectInvokable<IN, Tuple1<T0>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1> SingleOutputStreamOperator<Tuple2<T0, T1>, ?> types(Class<T0> type0,
- Class<T1> type1) {
- Class<?>[] types = { type0, type1 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple2<T0, T1>> outType = (TypeInformation<Tuple2<T0, T1>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream.transform("Projection", outType,
- new ProjectInvokable<IN, Tuple2<T0, T1>>(fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2> SingleOutputStreamOperator<Tuple3<T0, T1, T2>, ?> types(Class<T0> type0,
- Class<T1> type1, Class<T2> type2) {
- Class<?>[] types = { type0, type1, type2 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple3<T0, T1, T2>> outType = (TypeInformation<Tuple3<T0, T1, T2>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream.transform("Projection", outType,
- new ProjectInvokable<IN, Tuple3<T0, T1, T2>>(fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3> SingleOutputStreamOperator<Tuple4<T0, T1, T2, T3>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3) {
- Class<?>[] types = { type0, type1, type2, type3 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple4<T0, T1, T2, T3>> outType = (TypeInformation<Tuple4<T0, T1, T2, T3>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream.transform("Projection", outType,
- new ProjectInvokable<IN, Tuple4<T0, T1, T2, T3>>(fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4> SingleOutputStreamOperator<Tuple5<T0, T1, T2, T3, T4>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4) {
- Class<?>[] types = { type0, type1, type2, type3, type4 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple5<T0, T1, T2, T3, T4>> outType = (TypeInformation<Tuple5<T0, T1, T2, T3, T4>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream.transform("Projection", outType,
- new ProjectInvokable<IN, Tuple5<T0, T1, T2, T3, T4>>(fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5> SingleOutputStreamOperator<Tuple6<T0, T1, T2, T3, T4, T5>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple6<T0, T1, T2, T3, T4, T5>> outType = (TypeInformation<Tuple6<T0, T1, T2, T3, T4, T5>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream.transform("Projection", outType,
- new ProjectInvokable<IN, Tuple6<T0, T1, T2, T3, T4, T5>>(fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6> SingleOutputStreamOperator<Tuple7<T0, T1, T2, T3, T4, T5, T6>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple7<T0, T1, T2, T3, T4, T5, T6>> outType = (TypeInformation<Tuple7<T0, T1, T2, T3, T4, T5, T6>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform("Projection", outType,
- new ProjectInvokable<IN, Tuple7<T0, T1, T2, T3, T4, T5, T6>>(fieldIndexes,
- outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7> SingleOutputStreamOperator<Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>> outType = (TypeInformation<Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream.transform("Projection", outType,
- new ProjectInvokable<IN, Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>>(fieldIndexes,
- outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8> SingleOutputStreamOperator<Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>> outType = (TypeInformation<Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream.transform("Projection", outType,
- new ProjectInvokable<IN, Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>>(fieldIndexes,
- outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9> SingleOutputStreamOperator<Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>> outType = (TypeInformation<Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream.transform("Projection", outType,
- new ProjectInvokable<IN, Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10> SingleOutputStreamOperator<Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>> outType = (TypeInformation<Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream.transform("Projection", outType,
- new ProjectInvokable<IN, Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11> SingleOutputStreamOperator<Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>> outType = (TypeInformation<Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12> SingleOutputStreamOperator<Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>> outType = (TypeInformation<Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13> SingleOutputStreamOperator<Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>> outType = (TypeInformation<Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14> SingleOutputStreamOperator<Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>> outType = (TypeInformation<Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @param type15
- * The class of field '15' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15> SingleOutputStreamOperator<Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14, Class<T15> type15) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14, type15 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>> outType = (TypeInformation<Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @param type15
- * The class of field '15' of the result Tuples.
- * @param type16
- * The class of field '16' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16> SingleOutputStreamOperator<Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14, Class<T15> type15, Class<T16> type16) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14, type15, type16 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>> outType = (TypeInformation<Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @param type15
- * The class of field '15' of the result Tuples.
- * @param type16
- * The class of field '16' of the result Tuples.
- * @param type17
- * The class of field '17' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17> SingleOutputStreamOperator<Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14, type15, type16, type17 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>> outType = (TypeInformation<Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @param type15
- * The class of field '15' of the result Tuples.
- * @param type16
- * The class of field '16' of the result Tuples.
- * @param type17
- * The class of field '17' of the result Tuples.
- * @param type18
- * The class of field '18' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18> SingleOutputStreamOperator<Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
- Class<T18> type18) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14, type15, type16, type17, type18 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>> outType = (TypeInformation<Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @param type15
- * The class of field '15' of the result Tuples.
- * @param type16
- * The class of field '16' of the result Tuples.
- * @param type17
- * The class of field '17' of the result Tuples.
- * @param type18
- * The class of field '18' of the result Tuples.
- * @param type19
- * The class of field '19' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19> SingleOutputStreamOperator<Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
- Class<T18> type18, Class<T19> type19) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14, type15, type16, type17, type18, type19 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>> outType = (TypeInformation<Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @param type15
- * The class of field '15' of the result Tuples.
- * @param type16
- * The class of field '16' of the result Tuples.
- * @param type17
- * The class of field '17' of the result Tuples.
- * @param type18
- * The class of field '18' of the result Tuples.
- * @param type19
- * The class of field '19' of the result Tuples.
- * @param type20
- * The class of field '20' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20> SingleOutputStreamOperator<Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
- Class<T18> type18, Class<T19> type19, Class<T20> type20) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14, type15, type16, type17, type18, type19,
- type20 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>> outType = (TypeInformation<Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @param type15
- * The class of field '15' of the result Tuples.
- * @param type16
- * The class of field '16' of the result Tuples.
- * @param type17
- * The class of field '17' of the result Tuples.
- * @param type18
- * The class of field '18' of the result Tuples.
- * @param type19
- * The class of field '19' of the result Tuples.
- * @param type20
- * The class of field '20' of the result Tuples.
- * @param type21
- * The class of field '21' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21> SingleOutputStreamOperator<Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
- Class<T18> type18, Class<T19> type19, Class<T20> type20, Class<T21> type21) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14, type15, type16, type17, type18, type19,
- type20, type21 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>> outType = (TypeInformation<Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @param type15
- * The class of field '15' of the result Tuples.
- * @param type16
- * The class of field '16' of the result Tuples.
- * @param type17
- * The class of field '17' of the result Tuples.
- * @param type18
- * The class of field '18' of the result Tuples.
- * @param type19
- * The class of field '19' of the result Tuples.
- * @param type20
- * The class of field '20' of the result Tuples.
- * @param type21
- * The class of field '21' of the result Tuples.
- * @param type22
- * The class of field '22' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22> SingleOutputStreamOperator<Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
- Class<T18> type18, Class<T19> type19, Class<T20> type20, Class<T21> type21,
- Class<T22> type22) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14, type15, type16, type17, type18, type19,
- type20, type21, type22 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>> outType = (TypeInformation<Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @param type15
- * The class of field '15' of the result Tuples.
- * @param type16
- * The class of field '16' of the result Tuples.
- * @param type17
- * The class of field '17' of the result Tuples.
- * @param type18
- * The class of field '18' of the result Tuples.
- * @param type19
- * The class of field '19' of the result Tuples.
- * @param type20
- * The class of field '20' of the result Tuples.
- * @param type21
- * The class of field '21' of the result Tuples.
- * @param type22
- * The class of field '22' of the result Tuples.
- * @param type23
- * The class of field '23' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23> SingleOutputStreamOperator<Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
- Class<T18> type18, Class<T19> type19, Class<T20> type20, Class<T21> type21,
- Class<T22> type22, Class<T23> type23) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14, type15, type16, type17, type18, type19,
- type20, type21, type22, type23 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>> outType = (TypeInformation<Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
- outType,
- new ProjectInvokable<IN, Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>>(
- fieldIndexes, outType));
- }
-
- /**
- * Projects a {@link Tuple} {@link DataStream} to the previously selected
- * fields. Requires the classes of the fields of the resulting Tuples.
- *
- * @param type0
- * The class of field '0' of the result Tuples.
- * @param type1
- * The class of field '1' of the result Tuples.
- * @param type2
- * The class of field '2' of the result Tuples.
- * @param type3
- * The class of field '3' of the result Tuples.
- * @param type4
- * The class of field '4' of the result Tuples.
- * @param type5
- * The class of field '5' of the result Tuples.
- * @param type6
- * The class of field '6' of the result Tuples.
- * @param type7
- * The class of field '7' of the result Tuples.
- * @param type8
- * The class of field '8' of the result Tuples.
- * @param type9
- * The class of field '9' of the result Tuples.
- * @param type10
- * The class of field '10' of the result Tuples.
- * @param type11
- * The class of field '11' of the result Tuples.
- * @param type12
- * The class of field '12' of the result Tuples.
- * @param type13
- * The class of field '13' of the result Tuples.
- * @param type14
- * The class of field '14' of the result Tuples.
- * @param type15
- * The class of field '15' of the result Tuples.
- * @param type16
- * The class of field '16' of the result Tuples.
- * @param type17
- * The class of field '17' of the result Tuples.
- * @param type18
- * The class of field '18' of the result Tuples.
- * @param type19
- * The class of field '19' of the result Tuples.
- * @param type20
- * The class of field '20' of the result Tuples.
- * @param type21
- * The class of field '21' of the result Tuples.
- * @param type22
- * The class of field '22' of the result Tuples.
- * @param type23
- * The class of field '23' of the result Tuples.
- * @param type24
- * The class of field '24' of the result Tuples.
- * @return The projected DataStream.
- *
- * @see Tuple
- * @see DataStream
- */
- public <T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24> SingleOutputStreamOperator<Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>, ?> types(
- Class<T0> type0, Class<T1> type1, Class<T2> type2, Class<T3> type3, Class<T4> type4,
- Class<T5> type5, Class<T6> type6, Class<T7> type7, Class<T8> type8, Class<T9> type9,
- Class<T10> type10, Class<T11> type11, Class<T12> type12, Class<T13> type13,
- Class<T14> type14, Class<T15> type15, Class<T16> type16, Class<T17> type17,
- Class<T18> type18, Class<T19> type19, Class<T20> type20, Class<T21> type21,
- Class<T22> type22, Class<T23> type23, Class<T24> type24) {
- Class<?>[] types = { type0, type1, type2, type3, type4, type5, type6, type7, type8, type9,
- type10, type11, type12, type13, type14, type15, type16, type17, type18, type19,
- type20, type21, type22, type23, type24 };
- if (types.length != this.fieldIndexes.length) {
- throw new IllegalArgumentException(
- "Numbers of projected fields and types do not match.");
- }
-
- @SuppressWarnings("unchecked")
- TypeInformation<Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>> outType = (TypeInformation<Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>>) extractFieldTypes(
- fieldIndexes, types, inTypeInfo);
- return dataStream
- .transform(
- "Projection",
-
- outType,
- new ProjectInvokable<IN, Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>>(
- fieldIndexes, outType));
- }
-
- public static TypeInformation<?> extractFieldTypes(int[] fields, Class<?>[] givenTypes,
- TypeInformation<?> inType) {
-
- TupleTypeInfo<?> inTupleType = (TupleTypeInfo<?>) inType;
- TypeInformation<?>[] fieldTypes = new TypeInformation[fields.length];
-
- for (int i = 0; i < fields.length; i++) {
-
- if (inTupleType.getTypeAt(fields[i]).getTypeClass() != givenTypes[i]) {
- throw new IllegalArgumentException(
- "Given types do not match types of input data set.");
- }
-
- fieldTypes[i] = inTupleType.getTypeAt(fields[i]);
- }
-
- return new TupleTypeInfo<Tuple>(fieldTypes);
- }
-
-}
[37/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java
deleted file mode 100755
index 8afac75..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.flink.util.Collector;
-
-public class FromElementsFunction<T> implements SourceFunction<T> {
- private static final long serialVersionUID = 1L;
-
- Iterable<T> iterable;
-
- public FromElementsFunction(T... elements) {
- this.iterable = Arrays.asList(elements);
- }
-
- public FromElementsFunction(Collection<T> elements) {
- this.iterable = elements;
- }
-
- public FromElementsFunction(Iterable<T> elements) {
- this.iterable = elements;
- }
-
- @Override
- public void invoke(Collector<T> collector) throws Exception {
- for (T element : iterable) {
- collector.collect(element);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java
deleted file mode 100755
index 3afd06e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.NumberSequenceIterator;
-
-/**
- * Source Function used to generate the number sequence
- *
- */
-public class GenSequenceFunction extends RichParallelSourceFunction<Long> {
-
- private static final long serialVersionUID = 1L;
-
- private NumberSequenceIterator fullIterator;
- private NumberSequenceIterator splitIterator;
-
- public GenSequenceFunction(long from, long to) {
- fullIterator = new NumberSequenceIterator(from, to);
- }
-
- @Override
- public void invoke(Collector<Long> collector) throws Exception {
- while (splitIterator.hasNext()) {
- collector.collect(splitIterator.next());
- }
- }
-
- @Override
- public void open(Configuration config) {
- int splitNumber = getRuntimeContext().getIndexOfThisSubtask();
- int numOfSubTasks = getRuntimeContext().getNumberOfParallelSubtasks();
- splitIterator = fullIterator.split(numOfSubTasks)[splitNumber];
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java
deleted file mode 100644
index 664d39a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-
-public interface GenericSourceFunction<T> {
-
- public TypeInformation<T> getType();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
deleted file mode 100644
index 46d4fe9..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import org.apache.flink.util.Collector;
-
-public interface ParallelSourceFunction<OUT> extends SourceFunction<OUT> {
-
- public void invoke(Collector<OUT> collector) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java
deleted file mode 100644
index 5bbfd4c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import org.apache.flink.api.common.functions.AbstractRichFunction;
-
-public abstract class RichParallelSourceFunction<OUT> extends AbstractRichFunction implements
- ParallelSourceFunction<OUT> {
-
- private static final long serialVersionUID = 1L;
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java
deleted file mode 100755
index 4b947c7..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import org.apache.flink.api.common.functions.AbstractRichFunction;
-
-public abstract class RichSourceFunction<OUT> extends AbstractRichFunction implements
- SourceFunction<OUT> {
-
- private static final long serialVersionUID = 1L;
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SocketTextStreamFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SocketTextStreamFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SocketTextStreamFunction.java
deleted file mode 100644
index ac82b10..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SocketTextStreamFunction.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.util.Collector;
-
-public class SocketTextStreamFunction extends RichSourceFunction<String> {
- private static final long serialVersionUID = 1L;
-
- private String hostname;
- private int port;
- private char delimiter;
- private Socket socket;
- private static final int CONNECTION_TIMEOUT_TIME = 0;
-
- public SocketTextStreamFunction(String hostname, int port, char delimiter) {
- this.hostname = hostname;
- this.port = port;
- this.delimiter = delimiter;
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- socket = new Socket();
-
- socket.connect(new InetSocketAddress(hostname, port), CONNECTION_TIMEOUT_TIME);
- }
-
- @Override
- public void invoke(Collector<String> collector) throws Exception {
- while (!socket.isClosed() && socket.isConnected()) {
- streamFromSocket(collector, socket);
- }
- }
-
- public void streamFromSocket(Collector<String> collector, Socket socket) throws Exception {
- StringBuffer buffer = new StringBuffer();
- BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream()));
-
- while (true) {
- int data = reader.read();
- if (!socket.isConnected() || socket.isClosed() || data == -1) {
- break;
- }
-
- if (data == delimiter) {
- collector.collect(buffer.toString());
- buffer = new StringBuffer();
- } else if (data != '\r') { // ignore carriage return
- buffer.append((char) data);
- }
- }
-
- if (buffer.length() > 0) {
- collector.collect(buffer.toString());
- }
- }
-
- @Override
- public void close() throws Exception {
- socket.close();
- super.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SourceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SourceFunction.java
deleted file mode 100755
index 917562a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/SourceFunction.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.util.Collector;
-
-public interface SourceFunction<OUT> extends Function, Serializable {
-
- public void invoke(Collector<OUT> collector) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
deleted file mode 100644
index 24c0319..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.util.Collector;
-
-public abstract class ChainableInvokable<IN, OUT> extends StreamInvokable<IN, OUT> implements
- Collector<IN> {
-
- private static final long serialVersionUID = 1L;
-
- public ChainableInvokable(Function userFunction) {
- super(userFunction);
- setChainingStrategy(ChainingStrategy.ALWAYS);
- }
-
- public void setup(Collector<OUT> collector, StreamRecordSerializer<IN> inSerializer) {
- this.collector = collector;
- this.inSerializer = inSerializer;
- this.objectSerializer = inSerializer.getObjectSerializer();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
deleted file mode 100755
index 13a6ba1..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable;
-
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
-
-public class SinkInvokable<IN> extends ChainableInvokable<IN, IN> {
- private static final long serialVersionUID = 1L;
-
- private SinkFunction<IN> sinkFunction;
-
- public SinkInvokable(SinkFunction<IN> sinkFunction) {
- super(sinkFunction);
- this.sinkFunction = sinkFunction;
- }
-
- @Override
- public void invoke() throws Exception {
- while (readNext() != null) {
- callUserFunctionAndLogException();
- }
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- sinkFunction.invoke(nextObject);
- }
-
- @Override
- public void collect(IN record) {
- nextObject = copy(record);
- callUserFunctionAndLogException();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SourceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SourceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SourceInvokable.java
deleted file mode 100644
index f1cf2c5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SourceInvokable.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable;
-
-import java.io.Serializable;
-
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-
-public class SourceInvokable<OUT> extends StreamInvokable<OUT, OUT> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private SourceFunction<OUT> sourceFunction;
-
- public SourceInvokable(SourceFunction<OUT> sourceFunction) {
- super(sourceFunction);
- this.sourceFunction = sourceFunction;
- }
-
- @Override
- public void invoke() {
- callUserFunctionAndLogException();
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- sourceFunction.invoke(collector);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
deleted file mode 100644
index 6cee5f2..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable;
-
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.common.functions.util.FunctionUtils;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.api.streamvertex.StreamTaskContext;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.MutableObjectIterator;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The StreamInvokable represents the base class for all invokables in the
- * streaming topology.
- *
- * @param <OUT>
- * The output type of the invokable
- */
-public abstract class StreamInvokable<IN, OUT> implements Serializable {
-
- private static final long serialVersionUID = 1L;
- private static final Logger LOG = LoggerFactory.getLogger(StreamInvokable.class);
-
- protected StreamTaskContext<OUT> taskContext;
-
- protected MutableObjectIterator<StreamRecord<IN>> recordIterator;
- protected StreamRecordSerializer<IN> inSerializer;
- protected TypeSerializer<IN> objectSerializer;
- protected StreamRecord<IN> nextRecord;
- protected IN nextObject;
- protected boolean isMutable;
-
- protected Collector<OUT> collector;
- protected Function userFunction;
- protected volatile boolean isRunning;
-
- private ChainingStrategy chainingStrategy = ChainingStrategy.HEAD;
-
- public StreamInvokable(Function userFunction) {
- this.userFunction = userFunction;
- }
-
- /**
- * Initializes the {@link StreamInvokable} for input and output handling
- *
- * @param taskContext
- * StreamTaskContext representing the vertex
- */
- public void setup(StreamTaskContext<OUT> taskContext) {
- this.collector = taskContext.getOutputCollector();
- this.recordIterator = taskContext.getInput(0);
- this.inSerializer = taskContext.getInputSerializer(0);
- if (this.inSerializer != null) {
- this.nextRecord = inSerializer.createInstance();
- this.objectSerializer = inSerializer.getObjectSerializer();
- }
- this.taskContext = taskContext;
- }
-
- /**
- * Method that will be called when the operator starts, should encode the
- * processing logic
- */
- public abstract void invoke() throws Exception;
-
- /*
- * Reads the next record from the reader iterator and stores it in the
- * nextRecord variable
- */
- protected StreamRecord<IN> readNext() {
- this.nextRecord = inSerializer.createInstance();
- try {
- nextRecord = recordIterator.next(nextRecord);
- try {
- nextObject = nextRecord.getObject();
- } catch (NullPointerException e) {
- // end of stream
- }
- return nextRecord;
- } catch (IOException e) {
- throw new RuntimeException("Could not read next record.");
- }
- }
-
- /**
- * The call of the user implemented function should be implemented here
- */
- protected void callUserFunction() throws Exception {
- }
-
- /**
- * Method for logging exceptions thrown during the user function call
- */
- protected void callUserFunctionAndLogException() {
- try {
- callUserFunction();
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Calling user function failed due to: {}",
- StringUtils.stringifyException(e));
- }
- }
- }
-
- /**
- * Open method to be used if the user defined function extends the
- * RichFunction class
- *
- * @param parameters
- * The configuration parameters for the operator
- */
- public void open(Configuration parameters) throws Exception {
- isRunning = true;
- FunctionUtils.openFunction(userFunction, parameters);
- }
-
- /**
- * Close method to be used if the user defined function extends the
- * RichFunction class
- *
- */
- public void close() {
- isRunning = false;
- collector.close();
- try {
- FunctionUtils.closeFunction(userFunction);
- } catch (Exception e) {
- throw new RuntimeException("Error when closing the function: " + e.getMessage());
- }
- }
-
- public void setRuntimeContext(RuntimeContext t) {
- FunctionUtils.setFunctionRuntimeContext(userFunction, t);
- }
-
- protected IN copy(IN record) {
- return objectSerializer.copy(record);
- }
-
- public void setChainingStrategy(ChainingStrategy strategy) {
- if (strategy == ChainingStrategy.ALWAYS) {
- if (!(this instanceof ChainableInvokable)) {
- throw new RuntimeException(
- "Invokable needs to extend ChainableInvokable to be chained");
- }
- }
- this.chainingStrategy = strategy;
- }
-
- public ChainingStrategy getChainingStrategy() {
- return chainingStrategy;
- }
-
- public static enum ChainingStrategy {
- ALWAYS, NEVER, HEAD;
- }
-
- public Function getUserFunction() {
- return userFunction;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchIterator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchIterator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchIterator.java
deleted file mode 100755
index c3475e9..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchIterator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.io.Serializable;
-import java.util.Iterator;
-
-public interface BatchIterator<IN> extends Iterator<IN>, Serializable {
- public void reset();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
deleted file mode 100644
index 3fc314c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-
-public class CounterInvokable<IN> extends ChainableInvokable<IN, Long> {
- private static final long serialVersionUID = 1L;
-
- Long count = 0L;
-
- public CounterInvokable() {
- super(null);
- }
-
- @Override
- public void invoke() throws Exception {
- while (readNext() != null) {
- collector.collect(++count);
- }
- }
-
- @Override
- public void collect(IN record) {
- collector.collect(++count);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
deleted file mode 100644
index 0c8298e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-
-public class FilterInvokable<IN> extends ChainableInvokable<IN, IN> {
-
- private static final long serialVersionUID = 1L;
-
- FilterFunction<IN> filterFunction;
- private boolean collect;
-
- public FilterInvokable(FilterFunction<IN> filterFunction) {
- super(filterFunction);
- this.filterFunction = filterFunction;
- }
-
- @Override
- public void invoke() throws Exception {
- while (readNext() != null) {
- callUserFunctionAndLogException();
- }
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- collect = filterFunction.filter(copy(nextObject));
- if (collect) {
- collector.collect(nextObject);
- }
- }
-
- @Override
- public void collect(IN record) {
- nextObject = copy(record);
- callUserFunctionAndLogException();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
deleted file mode 100644
index 2a4081b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-
-public class FlatMapInvokable<IN, OUT> extends ChainableInvokable<IN, OUT> {
- private static final long serialVersionUID = 1L;
-
- private FlatMapFunction<IN, OUT> flatMapper;
-
- public FlatMapInvokable(FlatMapFunction<IN, OUT> flatMapper) {
- super(flatMapper);
- this.flatMapper = flatMapper;
- }
-
- @Override
- public void invoke() throws Exception {
- while (readNext() != null) {
- callUserFunctionAndLogException();
- }
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- flatMapper.flatMap(nextObject, collector);
- }
-
- @Override
- public void collect(IN record) {
- nextObject = copy(record);
- callUserFunctionAndLogException();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
deleted file mode 100755
index c2177fa..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-
-public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
- private static final long serialVersionUID = 1L;
-
- private KeySelector<IN, ?> keySelector;
- private Map<Object, IN> values;
- private IN reduced;
-
- public GroupedReduceInvokable(ReduceFunction<IN> reducer, KeySelector<IN, ?> keySelector) {
- super(reducer);
- this.keySelector = keySelector;
- values = new HashMap<Object, IN>();
- }
-
- @Override
- protected void reduce() throws Exception {
- Object key = nextRecord.getKey(keySelector);
- currentValue = values.get(key);
- nextValue = nextObject;
- if (currentValue != null) {
- callUserFunctionAndLogException();
- values.put(key, reduced);
- collector.collect(reduced);
- } else {
- values.put(key, nextValue);
- collector.collect(nextValue);
- }
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- reduced = reducer.reduce(currentValue, nextValue);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
deleted file mode 100644
index 997463c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
+++ /dev/null
@@ -1,506 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.NoSuchElementException;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-/**
- * This invokable allows windowing based on {@link TriggerPolicy} and
- * {@link EvictionPolicy} instances including their active and cloneable
- * versions. It is additionally aware of the creation of windows per group.
- *
- * A {@link KeySelector} is used to specify the key position or key extraction.
- * The {@link ReduceFunction} will be executed on each group separately.
- * Policies might either be centralized or distributed. It is not possible to
- * use central and distributed eviction policies at the same time. A distributed
- * policy have to be a {@link CloneableTriggerPolicy} or
- * {@link CloneableEvictionPolicy} as it will be cloned to have separated
- * instances for each group. At the startup time the distributed policies will
- * be stored as sample, and only clones of them will be used to maintain the
- * groups. Therefore, each group starts with the initial policy states.
- *
- * While a distributed policy only gets notified with the elements belonging to
- * the respective group, a centralized policy get notified with all arriving
- * elements. When a centralized trigger occurred, all groups get triggered. This
- * is done by submitting the element which caused the trigger as real element to
- * the groups it belongs to and as fake element to all other groups. Within the
- * groups the element might be further processed, causing more triggers,
- * prenotifications of active distributed policies and evictions like usual.
- *
- * Central policies can be instance of {@link ActiveTriggerPolicy} and also
- * implement the
- * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
- * method. Fake elements created on prenotification will be forwarded to all
- * groups. The {@link ActiveTriggerCallback} is also implemented in a way, that
- * it forwards/distributed calls all groups.
- *
- * @param <IN>
- * The type of input elements handled by this operator invokable.
- */
-public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
-
- /**
- * Auto-generated serial version UID
- */
- private static final long serialVersionUID = -3469545957144404137L;
-
- private KeySelector<IN, ?> keySelector;
- private Configuration parameters;
- private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies;
- private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies;
- private LinkedList<ActiveEvictionPolicy<IN>> activeCentralEvictionPolicies;
- private LinkedList<EvictionPolicy<IN>> centralEvictionPolicies;
- private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies;
- private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies;
- private Map<Object, WindowInvokable<IN, OUT>> windowingGroups;
- private LinkedList<Thread> activePolicyThreads;
- private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
- private LinkedList<WindowInvokable<IN, OUT>> deleteOrderForCentralEviction;
-
- /**
- * This constructor creates an instance of the grouped windowing invokable.
- *
- * A {@link KeySelector} is used to specify the key position or key
- * extraction. The {@link ReduceFunction} will be executed on each group
- * separately. Policies might either be centralized or distributed. It is
- * not possible to use central and distributed eviction policies at the same
- * time. A distributed policy have to be a {@link CloneableTriggerPolicy} or
- * {@link CloneableEvictionPolicy} as it will be cloned to have separated
- * instances for each group. At the startup time the distributed policies
- * will be stored as sample, and only clones of them will be used to
- * maintain the groups. Therefore, each group starts with the initial policy
- * states.
- *
- * While a distributed policy only gets notified with the elements belonging
- * to the respective group, a centralized policy get notified with all
- * arriving elements. When a centralized trigger occurred, all groups get
- * triggered. This is done by submitting the element which caused the
- * trigger as real element to the groups it belongs to and as fake element
- * to all other groups. Within the groups the element might be further
- * processed, causing more triggers, prenotifications of active distributed
- * policies and evictions like usual.
- *
- * Central policies can be instance of {@link ActiveTriggerPolicy} and also
- * implement the
- * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
- * method. Fake elements created on prenotification will be forwarded to all
- * groups. The {@link ActiveTriggerCallback} is also implemented in a way,
- * that it forwards/distributed calls all groups.
- *
- * @param userFunction
- * The user defined function.
- * @param keySelector
- * A key selector to extract the key for the groups from the
- * input data.
- * @param distributedTriggerPolicies
- * Trigger policies to be distributed and maintained individually
- * within each group.
- * @param distributedEvictionPolicies
- * Eviction policies to be distributed and maintained
- * individually within each group. Note that there cannot be
- * both, central and distributed eviction policies at the same
- * time.
- * @param centralTriggerPolicies
- * Trigger policies which will only exist once at a central
- * place. In case a central policy triggers, it will cause all
- * groups to be emitted. (Remark: Empty groups cannot be emitted.
- * If only one element is contained a group, this element itself
- * is returned as aggregated result.)
- * @param centralEvictionPolicies
- * Eviction which will only exist once at a central place. Note
- * that there cannot be both, central and distributed eviction
- * policies at the same time. The central eviction policy will
- * work on an simulated element buffer containing all elements no
- * matter which group they belong to.
- */
- public GroupedWindowInvokable(Function userFunction, KeySelector<IN, ?> keySelector,
- LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
- LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
- LinkedList<TriggerPolicy<IN>> centralTriggerPolicies,
- LinkedList<EvictionPolicy<IN>> centralEvictionPolicies) {
-
- super(userFunction);
-
- this.keySelector = keySelector;
-
- // handle the triggers
- if (centralTriggerPolicies != null) {
- this.centralTriggerPolicies = centralTriggerPolicies;
- this.activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
-
- for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
- if (trigger instanceof ActiveTriggerPolicy) {
- this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
- }
- }
- } else {
- this.centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
- }
-
- if (distributedTriggerPolicies != null) {
- this.distributedTriggerPolicies = distributedTriggerPolicies;
- } else {
- this.distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
- }
-
- if (distributedEvictionPolicies != null) {
- this.distributedEvictionPolicies = distributedEvictionPolicies;
- } else {
- this.distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
- }
-
- this.activeCentralEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
-
- if (centralEvictionPolicies != null) {
- this.centralEvictionPolicies = centralEvictionPolicies;
-
- for (EvictionPolicy<IN> eviction : centralEvictionPolicies) {
- if (eviction instanceof ActiveEvictionPolicy) {
- this.activeCentralEvictionPolicies.add((ActiveEvictionPolicy<IN>) eviction);
- }
- }
- } else {
- this.centralEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
- }
-
- this.windowingGroups = new HashMap<Object, WindowInvokable<IN, OUT>>();
- this.activePolicyThreads = new LinkedList<Thread>();
- this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
- this.deleteOrderForCentralEviction = new LinkedList<WindowInvokable<IN, OUT>>();
-
- // check that not both, central and distributed eviction, is used at the
- // same time.
- if (!this.centralEvictionPolicies.isEmpty() && !this.distributedEvictionPolicies.isEmpty()) {
- throw new UnsupportedOperationException(
- "You can only use either central or distributed eviction policies but not both at the same time.");
- }
-
- // Check that there is at least one trigger and one eviction policy
- if (this.centralEvictionPolicies.isEmpty() && this.distributedEvictionPolicies.isEmpty()) {
- throw new UnsupportedOperationException(
- "You have to define at least one eviction policy");
- }
- if (this.centralTriggerPolicies.isEmpty() && this.distributedTriggerPolicies.isEmpty()) {
- throw new UnsupportedOperationException(
- "You have to define at least one trigger policy");
- }
-
- }
-
- @Override
- public void invoke() throws Exception {
- // Prevent empty data streams
- if (readNext() == null) {
- throw new RuntimeException("DataStream must not be empty");
- }
-
- // Continuously run
- while (nextRecord != null) {
- WindowInvokable<IN, OUT> groupInvokable = windowingGroups.get(keySelector
- .getKey(nextRecord.getObject()));
- if (groupInvokable == null) {
- groupInvokable = makeNewGroup(nextRecord);
- }
-
- // Run the precalls for central active triggers
- for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
- Object[] result = trigger.preNotifyTrigger(nextRecord.getObject());
- for (Object in : result) {
-
- // If central eviction is used, handle it here
- if (!activeCentralEvictionPolicies.isEmpty()) {
- evictElements(centralActiveEviction(in));
- }
-
- // process in groups
- for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
- group.processFakeElement(in, trigger);
- checkForEmptyGroupBuffer(group);
- }
- }
- }
-
- // Process non-active central triggers
- for (TriggerPolicy<IN> triggerPolicy : centralTriggerPolicies) {
- if (triggerPolicy.notifyTrigger(nextRecord.getObject())) {
- currentTriggerPolicies.add(triggerPolicy);
- }
- }
-
- if (currentTriggerPolicies.isEmpty()) {
-
- // only add the element to its group
- groupInvokable.processRealElement(nextRecord.getObject());
- checkForEmptyGroupBuffer(groupInvokable);
-
- // If central eviction is used, handle it here
- if (!centralEvictionPolicies.isEmpty()) {
- evictElements(centralEviction(nextRecord.getObject(), false));
- deleteOrderForCentralEviction.add(groupInvokable);
- }
-
- } else {
-
- // call user function for all groups
- for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
- if (group == groupInvokable) {
- // process real with initialized policies
- group.processRealElement(nextRecord.getObject(), currentTriggerPolicies);
- } else {
- // process like a fake but also initialized with
- // policies
- group.externalTriggerFakeElement(nextRecord.getObject(),
- currentTriggerPolicies);
- }
-
- // remove group in case it has an empty buffer
- // checkForEmptyGroupBuffer(group);
- }
-
- // If central eviction is used, handle it here
- if (!centralEvictionPolicies.isEmpty()) {
- evictElements(centralEviction(nextRecord.getObject(), true));
- deleteOrderForCentralEviction.add(groupInvokable);
- }
- }
-
- // clear current trigger list
- currentTriggerPolicies.clear();
-
- // read next record
- readNext();
- }
-
- // Stop all remaining threads from policies
- for (Thread t : activePolicyThreads) {
- t.interrupt();
- }
-
- // finally trigger the buffer.
- for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
- group.emitFinalWindow(centralTriggerPolicies);
- }
-
- }
-
- /**
- * This method creates a new group. The method gets called in case an
- * element arrives which has a key which was not seen before. The method
- * created a nested {@link WindowInvokable} and therefore created clones of
- * all distributed trigger and eviction policies.
- *
- * @param element
- * The element which leads to the generation of a new group
- * (previously unseen key)
- * @throws Exception
- * In case the {@link KeySelector} throws an exception in
- * {@link KeySelector#getKey(Object)}, the exception is not
- * catched by this method.
- */
- @SuppressWarnings("unchecked")
- private WindowInvokable<IN, OUT> makeNewGroup(StreamRecord<IN> element) throws Exception {
- // clone the policies
- LinkedList<TriggerPolicy<IN>> clonedDistributedTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
- LinkedList<EvictionPolicy<IN>> clonedDistributedEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
- for (CloneableTriggerPolicy<IN> trigger : this.distributedTriggerPolicies) {
- clonedDistributedTriggerPolicies.add(trigger.clone());
- }
- for (CloneableEvictionPolicy<IN> eviction : this.distributedEvictionPolicies) {
- clonedDistributedEvictionPolicies.add(eviction.clone());
- }
-
- WindowInvokable<IN, OUT> groupInvokable;
- if (userFunction instanceof ReduceFunction) {
- groupInvokable = (WindowInvokable<IN, OUT>) new WindowReduceInvokable<IN>(
- (ReduceFunction<IN>) userFunction, clonedDistributedTriggerPolicies,
- clonedDistributedEvictionPolicies);
- } else {
- groupInvokable = new WindowGroupReduceInvokable<IN, OUT>(
- (GroupReduceFunction<IN, OUT>) userFunction, clonedDistributedTriggerPolicies,
- clonedDistributedEvictionPolicies);
- }
-
- groupInvokable.setup(taskContext);
- groupInvokable.open(this.parameters);
- windowingGroups.put(keySelector.getKey(element.getObject()), groupInvokable);
-
- return groupInvokable;
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- this.parameters = parameters;
- for (ActiveTriggerPolicy<IN> tp : activeCentralTriggerPolicies) {
- Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
- if (target != null) {
- Thread thread = new Thread(target);
- activePolicyThreads.add(thread);
- thread.start();
- }
- }
- };
-
- /**
- * This method is used to notify central eviction policies with a real
- * element.
- *
- * @param input
- * the real element to notify the eviction policy.
- * @param triggered
- * whether a central trigger occurred or not.
- * @return The number of elements to be deleted from the buffer.
- */
- private int centralEviction(IN input, boolean triggered) {
- // Process the evictions and take care of double evictions
- // In case there are multiple eviction policies present,
- // only the one with the highest return value is recognized.
- int currentMaxEviction = 0;
- for (EvictionPolicy<IN> evictionPolicy : centralEvictionPolicies) {
- // use temporary variable to prevent multiple calls to
- // notifyEviction
- int tmp = evictionPolicy.notifyEviction(input, triggered,
- deleteOrderForCentralEviction.size());
- if (tmp > currentMaxEviction) {
- currentMaxEviction = tmp;
- }
- }
- return currentMaxEviction;
- }
-
- /**
- * This method is used to notify active central eviction policies with a
- * fake element.
- *
- * @param input
- * the fake element to notify the active central eviction
- * policies.
- * @return The number of elements to be deleted from the buffer.
- */
- private int centralActiveEviction(Object input) {
- // Process the evictions and take care of double evictions
- // In case there are multiple eviction policies present,
- // only the one with the highest return value is recognized.
- int currentMaxEviction = 0;
- for (ActiveEvictionPolicy<IN> evictionPolicy : activeCentralEvictionPolicies) {
- // use temporary variable to prevent multiple calls to
- // notifyEviction
- int tmp = evictionPolicy.notifyEvictionWithFakeElement(input,
- deleteOrderForCentralEviction.size());
- if (tmp > currentMaxEviction) {
- currentMaxEviction = tmp;
- }
- }
- return currentMaxEviction;
- }
-
- /**
- * This method is used in central eviction to delete a given number of
- * elements from the buffer.
- *
- * @param numToEvict
- * number of elements to delete from the virtual central element
- * buffer.
- */
- private void evictElements(int numToEvict) {
- HashSet<WindowInvokable<IN, OUT>> usedGroups = new HashSet<WindowInvokable<IN, OUT>>();
- for (; numToEvict > 0; numToEvict--) {
- WindowInvokable<IN, OUT> currentGroup = deleteOrderForCentralEviction.getFirst();
- // Do the eviction
- currentGroup.evictFirst();
- // Remember groups which possibly have an empty buffer after the
- // eviction
- usedGroups.add(currentGroup);
- try {
- deleteOrderForCentralEviction.removeFirst();
- } catch (NoSuchElementException e) {
- // when buffer is empty, ignore exception and stop deleting
- break;
- }
-
- }
-
- // Remove groups with empty buffer
- for (WindowInvokable<IN, OUT> group : usedGroups) {
- checkForEmptyGroupBuffer(group);
- }
- }
-
- /**
- * Checks if the element buffer of a given windowing group is empty. If so,
- * the group will be deleted.
- *
- * @param group
- * The windowing group to be checked and and removed in case its
- * buffer is empty.
- */
- private void checkForEmptyGroupBuffer(WindowInvokable<IN, OUT> group) {
- if (group.isBufferEmpty()) {
- windowingGroups.remove(group);
- }
- }
-
- /**
- * This callback class allows to handle the callbacks done by threads
- * defined in active trigger policies
- *
- * @see ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)
- */
- private class WindowingCallback implements ActiveTriggerCallback {
- private ActiveTriggerPolicy<IN> policy;
-
- public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
- this.policy = policy;
- }
-
- @Override
- public void sendFakeElement(Object datapoint) {
-
- // If central eviction is used, handle it here
- if (!centralEvictionPolicies.isEmpty()) {
- evictElements(centralActiveEviction(datapoint));
- }
-
- // handle element in groups
- for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
- group.processFakeElement(datapoint, policy);
- checkForEmptyGroupBuffer(group);
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
deleted file mode 100644
index 7c8e577..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-
-public class MapInvokable<IN, OUT> extends ChainableInvokable<IN, OUT> {
- private static final long serialVersionUID = 1L;
-
- private MapFunction<IN, OUT> mapper;
-
- public MapInvokable(MapFunction<IN, OUT> mapper) {
- super(mapper);
- this.mapper = mapper;
- }
-
- @Override
- public void invoke() throws Exception {
- while (readNext() != null) {
- callUserFunctionAndLogException();
- }
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- collector.collect(mapper.map(nextObject));
- }
-
- @Override
- public void collect(IN record) {
- nextObject = copy(record);
- callUserFunctionAndLogException();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java
deleted file mode 100644
index c9d9e5a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-
-public class ProjectInvokable<IN, OUT extends Tuple> extends StreamInvokable<IN, OUT> {
- private static final long serialVersionUID = 1L;
-
- transient OUT outTuple;
- TypeSerializer<OUT> outTypeSerializer;
- int[] fields;
- int numFields;
-
- public ProjectInvokable(int[] fields, TypeInformation<OUT> outTypeInformation) {
- super(null);
- this.fields = fields;
- this.numFields = this.fields.length;
- this.outTypeSerializer = outTypeInformation.createSerializer();
- }
-
- @Override
- public void invoke() throws Exception {
- while (readNext() != null) {
- callUserFunctionAndLogException();
- }
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- for (int i = 0; i < this.numFields; i++) {
- outTuple.setField(nextRecord.getField(fields[i]), i);
- }
- collector.collect(outTuple);
- }
-
- @Override
- public void open(Configuration config) throws Exception {
- super.open(config);
- outTuple = outTypeSerializer.createInstance();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
deleted file mode 100644
index e1a56cc..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-
-public class StreamReduceInvokable<IN> extends ChainableInvokable<IN, IN> {
- private static final long serialVersionUID = 1L;
-
- protected ReduceFunction<IN> reducer;
- protected IN currentValue;
- protected IN nextValue;
-
- public StreamReduceInvokable(ReduceFunction<IN> reducer) {
- super(reducer);
- this.reducer = reducer;
- currentValue = null;
- }
-
- @Override
- public void invoke() throws Exception {
- while (readNext() != null) {
- reduce();
- }
- }
-
- protected void reduce() throws Exception {
- callUserFunctionAndLogException();
-
- }
-
- @Override
- protected void callUserFunction() throws Exception {
-
- nextValue = nextObject;
-
- if (currentValue != null) {
- currentValue = reducer.reduce(currentValue, nextValue);
- } else {
- currentValue = nextValue;
-
- }
- collector.collect(currentValue);
-
- }
-
- @Override
- public void collect(IN record) {
- nextObject = copy(record);
- callUserFunctionAndLogException();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
deleted file mode 100644
index b3fdfe8..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.LinkedList;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-public class WindowGroupReduceInvokable<IN, OUT> extends WindowInvokable<IN, OUT> {
-
- private static final long serialVersionUID = 1L;
- GroupReduceFunction<IN, OUT> reducer;
-
- public WindowGroupReduceInvokable(GroupReduceFunction<IN, OUT> userFunction,
- LinkedList<TriggerPolicy<IN>> triggerPolicies,
- LinkedList<EvictionPolicy<IN>> evictionPolicies) {
- super(userFunction, triggerPolicies, evictionPolicies);
- this.reducer = userFunction;
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- reducer.reduce(copyBuffer(), collector);
- }
-
- public LinkedList<IN> copyBuffer() {
- LinkedList<IN> copy = new LinkedList<IN>();
- for (IN element : buffer) {
- copy.add(copy(element));
- }
- return copy;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
deleted file mode 100644
index ea891c9..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
-
- /**
- * Auto-generated serial version UID
- */
- private static final long serialVersionUID = -8038984294071650730L;
-
- private LinkedList<TriggerPolicy<IN>> triggerPolicies;
- private LinkedList<EvictionPolicy<IN>> evictionPolicies;
- private LinkedList<ActiveTriggerPolicy<IN>> activeTriggerPolicies;
- private LinkedList<ActiveEvictionPolicy<IN>> activeEvictionPolicies;
- private LinkedList<Thread> activePolicyTreads;
- protected LinkedList<IN> buffer;
- private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
-
- /**
- * This constructor created a windowing invokable using trigger and eviction
- * policies.
- *
- * @param userFunction
- * The user defined {@link ReduceFunction}
- * @param triggerPolicies
- * A list of {@link TriggerPolicy}s and/or
- * {@link ActiveTriggerPolicy}s
- * @param evictionPolicies
- * A list of {@link EvictionPolicy}s and/or
- * {@link ActiveEvictionPolicy}s
- */
- public WindowInvokable(Function userFunction, LinkedList<TriggerPolicy<IN>> triggerPolicies,
- LinkedList<EvictionPolicy<IN>> evictionPolicies) {
- super(userFunction);
-
- this.triggerPolicies = triggerPolicies;
- this.evictionPolicies = evictionPolicies;
-
- activeTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
- for (TriggerPolicy<IN> tp : triggerPolicies) {
- if (tp instanceof ActiveTriggerPolicy) {
- activeTriggerPolicies.add((ActiveTriggerPolicy<IN>) tp);
- }
- }
-
- activeEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
- for (EvictionPolicy<IN> ep : evictionPolicies) {
- if (ep instanceof ActiveEvictionPolicy) {
- activeEvictionPolicies.add((ActiveEvictionPolicy<IN>) ep);
- }
- }
-
- this.activePolicyTreads = new LinkedList<Thread>();
- this.buffer = new LinkedList<IN>();
- this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
- }
-
- @Override
- public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
- super.open(parameters);
- for (ActiveTriggerPolicy<IN> tp : activeTriggerPolicies) {
- Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
- if (target != null) {
- Thread thread = new Thread(target);
- activePolicyTreads.add(thread);
- thread.start();
- }
- }
- };
-
- /**
- * This class allows the active trigger threads to call back and push fake
- * elements at any time.
- */
- private class WindowingCallback implements ActiveTriggerCallback {
- private ActiveTriggerPolicy<IN> policy;
-
- public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
- this.policy = policy;
- }
-
- @Override
- public void sendFakeElement(Object datapoint) {
- processFakeElement(datapoint, this.policy);
- }
-
- }
-
- @Override
- public void invoke() throws Exception {
-
- // Prevent empty data streams
- if (readNext() == null) {
- throw new RuntimeException("DataStream must not be empty");
- }
-
- // Continuously run
- while (nextRecord != null) {
- processRealElement(nextRecord.getObject());
-
- // Load next StreamRecord
- readNext();
- }
-
- // Stop all remaining threads from policies
- for (Thread t : activePolicyTreads) {
- t.interrupt();
- }
-
- // finally trigger the buffer.
- emitFinalWindow(null);
-
- }
-
- /**
- * This method gets called in case of an grouped windowing in case central
- * trigger occurred and the arriving element causing the trigger is not part
- * of this group.
- *
- * Remark: This is NOT the same as
- * {@link WindowInvokable#processFakeElement(Object, TriggerPolicy)}! Here
- * the eviction using active policies takes place after the call to the UDF.
- * Usually it is done before when fake elements get submitted. This special
- * behaviour is needed to allow the {@link GroupedWindowInvokable} to send
- * central triggers to all groups, even if the current element does not
- * belong to the group.
- *
- * @param input
- * a fake input element
- * @param policies
- * the list of policies which caused the call with this fake
- * element
- */
- protected synchronized void externalTriggerFakeElement(IN input,
- List<TriggerPolicy<IN>> policies) {
-
- // Set the current triggers
- currentTriggerPolicies.addAll(policies);
-
- // emit
- callUserFunctionAndLogException();
-
- // clear the flag collection
- currentTriggerPolicies.clear();
-
- // Process the evictions and take care of double evictions
- // In case there are multiple eviction policies present,
- // only the one with the highest return value is recognized.
- int currentMaxEviction = 0;
- for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
- // use temporary variable to prevent multiple calls to
- // notifyEviction
- int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
- if (tmp > currentMaxEviction) {
- currentMaxEviction = tmp;
- }
- }
-
- for (int i = 0; i < currentMaxEviction; i++) {
- try {
- buffer.removeFirst();
- } catch (NoSuchElementException e) {
- // In case no more elements are in the buffer:
- // Prevent failure and stop deleting.
- break;
- }
- }
- }
-
- /**
- * This method processed an arrived fake element The method is synchronized
- * to ensure that it cannot interleave with
- * {@link WindowInvokable#processRealElement(Object)}
- *
- * @param input
- * a fake input element
- * @param currentPolicy
- * the policy which produced this fake element
- */
- protected synchronized void processFakeElement(Object input, TriggerPolicy<IN> currentPolicy) {
-
- // Process the evictions and take care of double evictions
- // In case there are multiple eviction policies present,
- // only the one with the highest return value is recognized.
- int currentMaxEviction = 0;
- for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
- // use temporary variable to prevent multiple calls to
- // notifyEviction
- int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
- if (tmp > currentMaxEviction) {
- currentMaxEviction = tmp;
- }
- }
-
- for (int i = 0; i < currentMaxEviction; i++) {
- try {
- buffer.removeFirst();
- } catch (NoSuchElementException e) {
- // In case no more elements are in the buffer:
- // Prevent failure and stop deleting.
- break;
- }
- }
-
- // Set the current trigger
- currentTriggerPolicies.add(currentPolicy);
-
- // emit
- callUserFunctionAndLogException();
-
- // clear the flag collection
- currentTriggerPolicies.clear();
- }
-
- /**
- * This method processed an arrived real element The method is synchronized
- * to ensure that it cannot interleave with
- * {@link WindowInvokable#processFakeElement(Object)}.
- *
- * @param input
- * a real input element
- * @param triggerPolicies
- * Allows to set trigger policies which are maintained
- * externally. This is the case for central policies in
- * {@link GroupedWindowInvokable}.
- */
- protected synchronized void processRealElement(IN input, List<TriggerPolicy<IN>> triggerPolicies) {
- this.currentTriggerPolicies.addAll(triggerPolicies);
- processRealElement(input);
- }
-
- /**
- * This method processed an arrived real element The method is synchronized
- * to ensure that it cannot interleave with
- * {@link WindowInvokable#processFakeElement(Object)}
- *
- * @param input
- * a real input element
- */
- protected synchronized void processRealElement(IN input) {
-
- // Run the precalls to detect missed windows
- for (ActiveTriggerPolicy<IN> trigger : activeTriggerPolicies) {
- // Remark: In case multiple active triggers are present the ordering
- // of the different fake elements returned by this triggers becomes
- // a problem. This might lead to unexpected results...
- // Should we limit the number of active triggers to 0 or 1?
- Object[] result = trigger.preNotifyTrigger(input);
- for (Object in : result) {
- processFakeElement(in, trigger);
- }
- }
-
- // Remember if a trigger occurred
- boolean isTriggered = false;
-
- // Process the triggers
- for (TriggerPolicy<IN> triggerPolicy : triggerPolicies) {
- if (triggerPolicy.notifyTrigger(input)) {
- currentTriggerPolicies.add(triggerPolicy);
- }
- }
-
- // call user function
- if (!currentTriggerPolicies.isEmpty()) {
- // emit
- callUserFunctionAndLogException();
-
- // clear the flag collection
- currentTriggerPolicies.clear();
-
- // remember trigger
- isTriggered = true;
- }
-
- // Process the evictions and take care of double evictions
- // In case there are multiple eviction policies present,
- // only the one with the highest return value is recognized.
- int currentMaxEviction = 0;
-
- for (EvictionPolicy<IN> evictionPolicy : evictionPolicies) {
- // use temporary variable to prevent multiple calls to
- // notifyEviction
- int tmp = evictionPolicy.notifyEviction(input, isTriggered, buffer.size());
- if (tmp > currentMaxEviction) {
- currentMaxEviction = tmp;
- }
- }
-
- for (int i = 0; i < currentMaxEviction; i++) {
- try {
- buffer.removeFirst();
- } catch (NoSuchElementException e) {
- // In case no more elements are in the buffer:
- // Prevent failure and stop deleting.
- break;
- }
- }
-
- // Add the current element to the buffer
- buffer.add(input);
-
- }
-
- /**
- * This method removes the first element from the element buffer. It is used
- * to provide central evictions in {@link GroupedWindowInvokable}
- */
- protected synchronized void evictFirst() {
- try {
- buffer.removeFirst();
- } catch (NoSuchElementException e) {
- // ignore exception
- }
- }
-
- /**
- * This method returns whether the element buffer is empty or not. It is
- * used to figure out if a group can be deleted or not when
- * {@link GroupedWindowInvokable} is used.
- *
- * @return true in case the buffer is empty otherwise false.
- */
- protected boolean isBufferEmpty() {
- return buffer.isEmpty();
- }
-
- /**
- * This method does the final reduce at the end of the stream and emits the
- * result.
- *
- * @param centralTriggerPolicies
- * Allows to set trigger policies which are maintained
- * externally. This is the case for central policies in
- * {@link GroupedWindowInvokable}.
- */
- protected void emitFinalWindow(List<TriggerPolicy<IN>> centralTriggerPolicies) {
- if (!buffer.isEmpty()) {
- currentTriggerPolicies.clear();
-
- if (centralTriggerPolicies != null) {
- currentTriggerPolicies.addAll(centralTriggerPolicies);
- }
-
- for (TriggerPolicy<IN> policy : triggerPolicies) {
- currentTriggerPolicies.add(policy);
- }
-
- callUserFunctionAndLogException();
- }
- }
-
-}
[07/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FilterTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FilterTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FilterTest.java
new file mode 100644
index 0000000..403dd17
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FilterTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class FilterTest implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ static class MyFilter implements FilterFunction<Integer> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean filter(Integer value) throws Exception {
+ return value % 2 == 0;
+ }
+ }
+
+ @Test
+ public void test() {
+ FilterInvokable<Integer> invokable = new FilterInvokable<Integer>(new MyFilter());
+
+ List<Integer> expected = Arrays.asList(2, 4, 6);
+ List<Integer> actual = MockContext.createAndExecute(invokable, Arrays.asList(1, 2, 3, 4, 5, 6, 7));
+
+ assertEquals(expected, actual);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FlatMapTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FlatMapTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FlatMapTest.java
new file mode 100644
index 0000000..7424e21
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/FlatMapTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.streaming.util.MockContext;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class FlatMapTest {
+
+ public static final class MyFlatMap implements FlatMapFunction<Integer, Integer> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(Integer value, Collector<Integer> out) throws Exception {
+ if (value % 2 == 0) {
+ out.collect(value);
+ out.collect(value * value);
+ }
+ }
+ }
+
+ @Test
+ public void flatMapTest() {
+ FlatMapInvokable<Integer, Integer> invokable = new FlatMapInvokable<Integer, Integer>(new MyFlatMap());
+
+ List<Integer> expected = Arrays.asList(2, 4, 4, 16, 6, 36, 8, 64);
+ List<Integer> actual = MockContext.createAndExecute(invokable, Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8));
+
+ assertEquals(expected, actual);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java
new file mode 100755
index 0000000..ce47c67
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class GroupedReduceInvokableTest {
+
+ private static class MyReducer implements ReduceFunction<Integer> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+
+ }
+
+ @Test
+ public void test() {
+ GroupedReduceInvokable<Integer> invokable1 = new GroupedReduceInvokable<Integer>(
+ new MyReducer(), new KeySelector<Integer, Integer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) throws Exception {
+ return value;
+ }
+ });
+
+ List<Integer> expected = Arrays.asList(1, 2, 2, 4, 3);
+ List<Integer> actual = MockContext.createAndExecute(invokable1,
+ Arrays.asList(1, 1, 2, 2, 3));
+
+ assertEquals(expected, actual);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
new file mode 100644
index 0000000..f38d5c1
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
@@ -0,0 +1,574 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class GroupedWindowInvokableTest {
+
+ KeySelector<Tuple2<Integer, String>, ?> keySelector = new KeySelector<Tuple2<Integer, String>, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple2<Integer, String> value) throws Exception {
+ return value.f1;
+ }
+ };
+
+ /**
+ * Tests that illegal arguments result in failure. The following cases are
+ * tested: 1) having no trigger 2) having no eviction 3) having neither
+ * eviction nor trigger 4) having both, central and distributed eviction.
+ */
+ @Test
+ public void testGroupedWindowInvokableFailTest() {
+
+ // create dummy reduce function
+ ReduceFunction<Object> userFunction = new ReduceFunction<Object>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Object reduce(Object value1, Object value2) throws Exception {
+ return null;
+ }
+ };
+
+ // create dummy keySelector
+ KeySelector<Object, Object> keySelector = new KeySelector<Object, Object>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Object getKey(Object value) throws Exception {
+ return null;
+ }
+ };
+
+ // create policy lists
+ LinkedList<CloneableEvictionPolicy<Object>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<Object>>();
+ LinkedList<CloneableTriggerPolicy<Object>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<Object>>();
+ LinkedList<EvictionPolicy<Object>> centralEvictionPolicies = new LinkedList<EvictionPolicy<Object>>();
+ LinkedList<TriggerPolicy<Object>> centralTriggerPolicies = new LinkedList<TriggerPolicy<Object>>();
+
+ // empty trigger and policy lists should fail
+ try {
+ new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
+ distributedTriggerPolicies, distributedEvictionPolicies,
+ centralTriggerPolicies, centralEvictionPolicies);
+ fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (1)");
+ } catch (UnsupportedOperationException e) {
+ // that's the expected case
+ }
+
+ // null for trigger and policy lists should fail
+ try {
+ new GroupedWindowInvokable<Object, Object>(userFunction, keySelector, null, null, null,
+ null);
+ fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (2)");
+ } catch (UnsupportedOperationException e) {
+ // that's the expected case
+ }
+
+ // empty eviction should still fail
+ centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
+ distributedTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
+ try {
+ new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
+ distributedTriggerPolicies, distributedEvictionPolicies,
+ centralTriggerPolicies, centralEvictionPolicies);
+ fail("Creating instance without any eviction policy should cause an UnsupportedOperationException but didn't. (3)");
+ } catch (UnsupportedOperationException e) {
+ // that's the expected case
+ }
+
+ // empty trigger should still fail
+ centralTriggerPolicies.clear();
+ distributedTriggerPolicies.clear();
+ centralEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
+ try {
+ new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
+ distributedTriggerPolicies, distributedEvictionPolicies,
+ centralTriggerPolicies, centralEvictionPolicies);
+ fail("Creating instance without any trigger policy should cause an UnsupportedOperationException but didn't. (4)");
+ } catch (UnsupportedOperationException e) {
+ // that's the expected case
+ }
+
+ // having both, central and distributed eviction, at the same time
+ // should fail
+ centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
+ distributedEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
+ try {
+ new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
+ distributedTriggerPolicies, distributedEvictionPolicies,
+ centralTriggerPolicies, centralEvictionPolicies);
+ fail("Creating instance with central and distributed eviction should cause an UnsupportedOperationException but didn't. (4)");
+ } catch (UnsupportedOperationException e) {
+ // that's the expected case
+ }
+
+ }
+
+ /**
+ * Test for not active distributed triggers with single field
+ */
+ @Test
+ public void testGroupedWindowInvokableDistributedTriggerSimple() {
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(1);
+ inputs.add(5);
+ inputs.add(5);
+ inputs.add(5);
+ inputs.add(1);
+ inputs.add(1);
+ inputs.add(5);
+ inputs.add(1);
+ inputs.add(5);
+
+ List<Integer> expectedDistributedEviction = new ArrayList<Integer>();
+ expectedDistributedEviction.add(15);
+ expectedDistributedEviction.add(3);
+ expectedDistributedEviction.add(3);
+ expectedDistributedEviction.add(15);
+
+ List<Integer> expectedCentralEviction = new ArrayList<Integer>();
+ expectedCentralEviction.add(2);
+ expectedCentralEviction.add(5);
+ expectedCentralEviction.add(15);
+ expectedCentralEviction.add(2);
+ expectedCentralEviction.add(5);
+ expectedCentralEviction.add(2);
+ expectedCentralEviction.add(5);
+ expectedCentralEviction.add(1);
+ expectedCentralEviction.add(5);
+
+ LinkedList<CloneableTriggerPolicy<Integer>> triggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+ // Trigger on every 2nd element, but the first time after the 3rd
+ triggers.add(new CountTriggerPolicy<Integer>(2, -1));
+
+ LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+ // On every 2nd element, remove the oldest 2 elements, but the first
+ // time after the 3rd element
+ evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+ LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
+
+ ReduceFunction<Integer> reduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) {
+ return value;
+ }
+ };
+
+ GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
+ reduceFunction, keySelector, triggers, evictions, centralTriggers, null);
+
+ List<Integer> result = MockContext.createAndExecute(invokable, inputs);
+
+ List<Integer> actual = new LinkedList<Integer>();
+ for (Integer current : result) {
+ actual.add(current);
+ }
+
+ assertEquals(new HashSet<Integer>(expectedDistributedEviction),
+ new HashSet<Integer>(actual));
+ assertEquals(expectedDistributedEviction.size(), actual.size());
+
+ // Run test with central eviction
+ triggers.clear();
+ centralTriggers.add(new CountTriggerPolicy<Integer>(2, -1));
+ LinkedList<EvictionPolicy<Integer>> centralEvictions = new LinkedList<EvictionPolicy<Integer>>();
+ centralEvictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+ invokable = new GroupedWindowInvokable<Integer, Integer>(reduceFunction, keySelector,
+ triggers, null, centralTriggers, centralEvictions);
+
+ result = MockContext.createAndExecute(invokable, inputs);
+ actual = new LinkedList<Integer>();
+ for (Integer current : result) {
+ actual.add(current);
+ }
+
+ assertEquals(new HashSet<Integer>(expectedCentralEviction), new HashSet<Integer>(actual));
+ assertEquals(expectedCentralEviction.size(), actual.size());
+ }
+
+ /**
+ * Test for non active distributed triggers with separated key field
+ */
+ @Test
+ public void testGroupedWindowInvokableDistributedTriggerComplex() {
+ List<Tuple2<Integer, String>> inputs2 = new ArrayList<Tuple2<Integer, String>>();
+ inputs2.add(new Tuple2<Integer, String>(1, "a"));
+ inputs2.add(new Tuple2<Integer, String>(0, "b"));
+ inputs2.add(new Tuple2<Integer, String>(2, "a"));
+ inputs2.add(new Tuple2<Integer, String>(-1, "a"));
+ inputs2.add(new Tuple2<Integer, String>(-2, "a"));
+ inputs2.add(new Tuple2<Integer, String>(10, "a"));
+ inputs2.add(new Tuple2<Integer, String>(2, "b"));
+ inputs2.add(new Tuple2<Integer, String>(1, "a"));
+
+ List<Tuple2<Integer, String>> expected2 = new ArrayList<Tuple2<Integer, String>>();
+ expected2.add(new Tuple2<Integer, String>(-1, "a"));
+ expected2.add(new Tuple2<Integer, String>(-2, "a"));
+ expected2.add(new Tuple2<Integer, String>(0, "b"));
+
+ LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
+ // Trigger on every 2nd element, but the first time after the 3rd
+ triggers.add(new CountTriggerPolicy<Tuple2<Integer, String>>(3));
+
+ LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
+ // On every 2nd element, remove the oldest 2 elements, but the first
+ // time after the 3rd element
+ evictions.add(new TumblingEvictionPolicy<Tuple2<Integer, String>>());
+
+ LinkedList<TriggerPolicy<Tuple2<Integer, String>>> centralTriggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
+
+ GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable2 = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+ new ReduceFunction<Tuple2<Integer, String>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
+ Tuple2<Integer, String> value2) throws Exception {
+ if (value1.f0 <= value2.f0) {
+ return value1;
+ } else {
+ return value2;
+ }
+ }
+ }, keySelector, triggers, evictions, centralTriggers, null);
+
+ List<Tuple2<Integer, String>> result = MockContext.createAndExecute(invokable2, inputs2);
+
+ List<Tuple2<Integer, String>> actual2 = new LinkedList<Tuple2<Integer, String>>();
+ for (Tuple2<Integer, String> current : result) {
+ actual2.add(current);
+ }
+
+ assertEquals(new HashSet<Tuple2<Integer, String>>(expected2),
+ new HashSet<Tuple2<Integer, String>>(actual2));
+ assertEquals(expected2.size(), actual2.size());
+ }
+
+ /**
+ * Test for active centralized trigger
+ */
+ @Test
+ public void testGroupedWindowInvokableCentralActiveTrigger() {
+
+ List<Tuple2<Integer, String>> inputs = new ArrayList<Tuple2<Integer, String>>();
+ inputs.add(new Tuple2<Integer, String>(1, "a"));
+ inputs.add(new Tuple2<Integer, String>(1, "b"));
+ inputs.add(new Tuple2<Integer, String>(1, "c"));
+ inputs.add(new Tuple2<Integer, String>(2, "a"));
+ inputs.add(new Tuple2<Integer, String>(2, "b"));
+ inputs.add(new Tuple2<Integer, String>(2, "c"));
+ inputs.add(new Tuple2<Integer, String>(2, "b"));
+ inputs.add(new Tuple2<Integer, String>(2, "a"));
+ inputs.add(new Tuple2<Integer, String>(2, "c"));
+ inputs.add(new Tuple2<Integer, String>(3, "c"));
+ inputs.add(new Tuple2<Integer, String>(3, "a"));
+ inputs.add(new Tuple2<Integer, String>(3, "b"));
+ inputs.add(new Tuple2<Integer, String>(4, "a"));
+ inputs.add(new Tuple2<Integer, String>(4, "b"));
+ inputs.add(new Tuple2<Integer, String>(4, "c"));
+ inputs.add(new Tuple2<Integer, String>(5, "c"));
+ inputs.add(new Tuple2<Integer, String>(5, "a"));
+ inputs.add(new Tuple2<Integer, String>(5, "b"));
+ inputs.add(new Tuple2<Integer, String>(10, "b"));
+ inputs.add(new Tuple2<Integer, String>(10, "a"));
+ inputs.add(new Tuple2<Integer, String>(10, "c"));
+ inputs.add(new Tuple2<Integer, String>(11, "a"));
+ inputs.add(new Tuple2<Integer, String>(11, "a"));
+ inputs.add(new Tuple2<Integer, String>(11, "c"));
+ inputs.add(new Tuple2<Integer, String>(11, "c"));
+ inputs.add(new Tuple2<Integer, String>(11, "b"));
+ inputs.add(new Tuple2<Integer, String>(11, "b"));
+
+ // Expected result:
+ // For each group (a,b and c):
+ // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
+ // 12-12-5-10-32
+
+ List<Tuple2<Integer, String>> expected = new ArrayList<Tuple2<Integer, String>>();
+ expected.add(new Tuple2<Integer, String>(12, "a"));
+ expected.add(new Tuple2<Integer, String>(12, "b"));
+ expected.add(new Tuple2<Integer, String>(12, "c"));
+ expected.add(new Tuple2<Integer, String>(12, "a"));
+ expected.add(new Tuple2<Integer, String>(12, "b"));
+ expected.add(new Tuple2<Integer, String>(12, "c"));
+ expected.add(new Tuple2<Integer, String>(5, "a"));
+ expected.add(new Tuple2<Integer, String>(5, "b"));
+ expected.add(new Tuple2<Integer, String>(5, "c"));
+ expected.add(new Tuple2<Integer, String>(10, "a"));
+ expected.add(new Tuple2<Integer, String>(10, "b"));
+ expected.add(new Tuple2<Integer, String>(10, "c"));
+ expected.add(new Tuple2<Integer, String>(32, "a"));
+ expected.add(new Tuple2<Integer, String>(32, "b"));
+ expected.add(new Tuple2<Integer, String>(32, "c"));
+
+ Timestamp<Tuple2<Integer, String>> myTimeStamp = new Timestamp<Tuple2<Integer, String>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(Tuple2<Integer, String> value) {
+ return value.f0;
+ }
+ };
+
+ TimestampWrapper<Tuple2<Integer, String>> myTimeStampWrapper = new TimestampWrapper<Tuple2<Integer, String>>(
+ myTimeStamp, 1);
+
+ ReduceFunction<Tuple2<Integer, String>> myReduceFunction = new ReduceFunction<Tuple2<Integer, String>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
+ Tuple2<Integer, String> value2) throws Exception {
+ return new Tuple2<Integer, String>(value1.f0 + value2.f0, value1.f1);
+ }
+ };
+
+ LinkedList<TriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
+ // Trigger every 2 time units but delay the first trigger by 2 (First
+ // trigger after 4, then every 2)
+ triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
+
+ LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
+ // Always delete all elements older then 4
+ evictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
+
+ LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
+
+ GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+ myReduceFunction, keySelector, distributedTriggers, evictions, triggers, null);
+
+ ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
+ for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
+ new HashSet<Tuple2<Integer, String>>(result));
+ assertEquals(expected.size(), result.size());
+
+ // repeat the test with central eviction. The result should be the same.
+ triggers.clear();
+ triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
+ evictions.clear();
+ LinkedList<EvictionPolicy<Tuple2<Integer, String>>> centralEvictions = new LinkedList<EvictionPolicy<Tuple2<Integer, String>>>();
+ centralEvictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
+
+ invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+ myReduceFunction, keySelector, distributedTriggers, evictions, triggers,
+ centralEvictions);
+
+ result = new ArrayList<Tuple2<Integer, String>>();
+ for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
+ new HashSet<Tuple2<Integer, String>>(result));
+ assertEquals(expected.size(), result.size());
+ }
+
+ /**
+ * Test for multiple centralized trigger
+ */
+ @Test
+ public void testGroupedWindowInvokableMultipleCentralTrigger() {
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ triggers.add(new CountTriggerPolicy<Integer>(8));
+ triggers.add(new CountTriggerPolicy<Integer>(5));
+
+ LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+ // The active wrapper causes eviction even on (fake) elements which
+ // triggered, but does not belong to the group.
+ evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
+ new TumblingEvictionPolicy<Integer>()));
+
+ LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(2);
+ inputs.add(2);
+ inputs.add(2);
+ inputs.add(1);
+ // 1st Trigger: 2;6
+ inputs.add(2);
+ inputs.add(1);
+ inputs.add(2);
+ // 2nd Trigger: 1;4
+ inputs.add(2);
+ inputs.add(1);
+ // Final: 1,2
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(2);
+ expected.add(6);
+ expected.add(4);
+ expected.add(1);
+ expected.add(2);
+ expected.add(1);
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
+ myReduceFunction, new KeySelector<Integer, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) {
+ return value;
+ }
+ }, distributedTriggers, evictions, triggers, null);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
+ assertEquals(expected.size(), result.size());
+ }
+
+ /**
+ * Test for combination of centralized trigger and distributed trigger at
+ * the same time
+ */
+ @Test
+ public void testGroupedWindowInvokableCentralAndDistrTrigger() {
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ triggers.add(new CountTriggerPolicy<Integer>(8));
+ triggers.add(new CountTriggerPolicy<Integer>(5));
+
+ LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+ // The active wrapper causes eviction even on (fake) elements which
+ // triggered, but does not belong to the group.
+ evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
+ new TumblingEvictionPolicy<Integer>()));
+
+ LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+ distributedTriggers.add(new CountTriggerPolicy<Integer>(2));
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(2);
+ inputs.add(2);
+ // local on 2 => 4
+ inputs.add(2);
+ inputs.add(1);
+ // local on 1 => 2
+ // and 1st Central: 2;2
+ // SUMS up to 2;2
+ inputs.add(2);
+ // local on 2 => 2
+ inputs.add(1);
+ inputs.add(2);
+ // 2nd Central: 1;2
+ inputs.add(2);
+ inputs.add(1);
+ // Final: 1,2
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(4);
+ expected.add(2);
+ expected.add(2);
+ expected.add(2);
+ expected.add(1);
+ expected.add(2);
+ expected.add(1);
+ expected.add(2);
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
+ myReduceFunction, new KeySelector<Integer, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) {
+ return value;
+ }
+ }, distributedTriggers, evictions, triggers, null);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
+ assertEquals(expected.size(), result.size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/MapTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/MapTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/MapTest.java
new file mode 100644
index 0000000..5390ec9
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/MapTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class MapTest {
+
+ private static class Map implements MapFunction<Integer, String> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String map(Integer value) throws Exception {
+ return "+" + (value + 1);
+ }
+ }
+
+ @Test
+ public void mapInvokableTest() {
+ MapInvokable<Integer, String> invokable = new MapInvokable<Integer, String>(new Map());
+
+ List<String> expectedList = Arrays.asList("+2", "+3", "+4");
+ List<String> actualList = MockContext.createAndExecute(invokable, Arrays.asList(1, 2, 3));
+
+ assertEquals(expectedList, actualList);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/ProjectTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/ProjectTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/ProjectTest.java
new file mode 100644
index 0000000..11c44cd
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/ProjectTest.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.datastream.StreamProjection;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class ProjectTest implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Test
+ public void test() {
+
+ TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
+ .getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b",
+ 4));
+
+ int[] fields = new int[] { 4, 4, 3 };
+ Class<?>[] classes = new Class<?>[] { Integer.class, Integer.class, String.class };
+
+ @SuppressWarnings("unchecked")
+ ProjectInvokable<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> invokable = new ProjectInvokable<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
+ fields,
+ (TypeInformation<Tuple3<Integer, Integer, String>>) StreamProjection
+ .extractFieldTypes(fields, classes, inType));
+
+ List<Tuple5<Integer, String, Integer, String, Integer>> input = new ArrayList<Tuple5<Integer, String, Integer, String, Integer>>();
+ input.add(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));
+ input.add(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2));
+ input.add(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2));
+ input.add(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7));
+
+ List<Tuple3<Integer, Integer, String>> expected = new ArrayList<Tuple3<Integer, Integer, String>>();
+ expected.add(new Tuple3<Integer, Integer, String>(4, 4, "b"));
+ expected.add(new Tuple3<Integer, Integer, String>(2, 2, "c"));
+ expected.add(new Tuple3<Integer, Integer, String>(2, 2, "c"));
+ expected.add(new Tuple3<Integer, Integer, String>(7, 7, "a"));
+
+ assertEquals(expected, MockContext.createAndExecute(invokable, input));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceTest.java
new file mode 100755
index 0000000..ae866e6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class StreamReduceTest {
+
+ private static class MyReducer implements ReduceFunction<Integer>{
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1+value2;
+ }
+
+ }
+
+ @Test
+ public void test() {
+ StreamReduceInvokable<Integer> invokable1 = new StreamReduceInvokable<Integer>(
+ new MyReducer());
+
+ List<Integer> expected = Arrays.asList(1,2,4,7,10);
+ List<Integer> actual = MockContext.createAndExecute(invokable1,
+ Arrays.asList(1, 1, 2, 3, 3));
+
+ assertEquals(expected, actual);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
new file mode 100644
index 0000000..83b4596
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class WindowInvokableTest {
+
+ /**
+ * Test case equal to {@link WindowReduceInvokableTest}
+ */
+ @Test
+ public void testWindowInvokableWithTimePolicy() {
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(2);
+ inputs.add(2);
+ inputs.add(3);
+ inputs.add(4);
+ inputs.add(5);
+ inputs.add(10);
+ inputs.add(11);
+ inputs.add(11);
+ // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
+ // 12-12-5-10-32
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(12);
+ expected.add(12);
+ expected.add(5);
+ expected.add(10);
+ expected.add(32);
+
+ Timestamp<Integer> myTimeStamp = new Timestamp<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(Integer value) {
+ return value;
+ }
+ };
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ // Trigger every 2 time units but delay the first trigger by 2 (First
+ // trigger after 4, then every 2)
+ triggers.add(new TimeTriggerPolicy<Integer>(2L, new TimestampWrapper<Integer>(myTimeStamp,
+ 1), 2L));
+
+ LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+ // Always delete all elements older then 4
+ evictions.add(new TimeEvictionPolicy<Integer>(4L, new TimestampWrapper<Integer>(
+ myTimeStamp, 1)));
+
+ WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(expected, result);
+ }
+
+ /**
+ * Test case equal to {@link BatchReduceTest}
+ */
+ @Test
+ public void testWindowInvokableWithCountPolicy() {
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ for (Integer i = 1; i <= 10; i++) {
+ inputs.add(i);
+ }
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ /*
+ * The following setup reassembles the batch size 3 and the slide size 2
+ * of the BatchReduceInvokable.
+ */
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ // Trigger on every 2nd element, but the first time after the 3rd
+ triggers.add(new CountTriggerPolicy<Integer>(2, -1));
+
+ LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+ // On every 2nd element, remove the oldest 2 elements, but the first
+ // time after the 3rd element
+ evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+ WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(6);
+ expected.add(12);
+ expected.add(18);
+ expected.add(24);
+ expected.add(19);
+ List<Integer> result = new ArrayList<Integer>();
+ for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+ assertEquals(expected, result);
+
+ /*
+ * Begin test part 2
+ */
+
+ List<Integer> inputs2 = new ArrayList<Integer>();
+ inputs2.add(1);
+ inputs2.add(2);
+ inputs2.add(-5); // changed this value to make sure it is excluded from
+ // the result
+ inputs2.add(-3);
+ inputs2.add(-4);
+
+ myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ if (value1 <= value2) {
+ return value1;
+ } else {
+ return value2;
+ }
+ };
+ };
+
+ /*
+ * The following setup reassembles the batch size 2 and the slide size 3
+ * of the BatchReduceInvokable.
+ */
+ triggers = new LinkedList<TriggerPolicy<Integer>>();
+ // Trigger after every 3rd element, but the first time after the 2nd
+ triggers.add(new CountTriggerPolicy<Integer>(3, 1));
+
+ evictions = new LinkedList<EvictionPolicy<Integer>>();
+ // On every 3rd element, remove the oldest 3 elements, but the first
+ // time after on the 5th element
+ evictions.add(new CountEvictionPolicy<Integer>(3, 3, -1));
+
+ WindowInvokable<Integer, Integer> invokable2 = new WindowReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
+
+ List<Integer> expected2 = new ArrayList<Integer>();
+ expected2.add(1);
+ expected2.add(-4);
+
+ result = new ArrayList<Integer>();
+ for (Integer t : MockContext.createAndExecute(invokable2, inputs2)) {
+ result.add(t);
+ }
+
+ assertEquals(expected2, result);
+
+ }
+
+ @Test
+ public void testWindowInvokableWithMultiplePolicies() {
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ triggers.add(new CountTriggerPolicy<Integer>(2));
+ triggers.add(new CountTriggerPolicy<Integer>(3));
+
+ LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+ evictions.add(new CountEvictionPolicy<Integer>(2, 2));
+ evictions.add(new CountEvictionPolicy<Integer>(3, 3));
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ for (Integer i = 1; i <= 10; i++) {
+ inputs.add(i);
+ }
+ /**
+ * <code>
+ * VAL: 1,2,3,4,5,6,7,8,9,10
+ * TR1: | | | | |
+ * TR2: | | |
+ * EV1: 2 2 2 2 2
+ * EV2: 3 3 3
+ * </code>
+ */
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(3);
+ expected.add(3);
+ expected.add(4);
+ expected.add(11);
+ expected.add(15);
+ expected.add(9);
+ expected.add(10);
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(expected, result);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamrecord/UIDTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamrecord/UIDTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamrecord/UIDTest.java
new file mode 100644
index 0000000..b044d88
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamrecord/UIDTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamrecord;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.flink.runtime.util.DataInputDeserializer;
+import org.apache.flink.runtime.util.DataOutputSerializer;
+import org.junit.Test;
+
+public class UIDTest {
+
+ //TODO fix with matching DataOutputStream and DataOutputView
+ @Test
+ public void test() throws IOException {
+ DataOutputSerializer out = new DataOutputSerializer(64);
+
+ UID id = new UID(3);
+ id.write(out);
+
+ ByteBuffer buff = out.wrapAsByteBuffer();
+
+
+ DataInputDeserializer in = new DataInputDeserializer(buff);
+
+ UID id2 = new UID();
+ id2.read(in);
+
+ assertEquals(id.getChannelId(), id2.getChannelId());
+ assertArrayEquals(id.getGeneratedId(), id2.getGeneratedId());
+ assertArrayEquals(id.getId(), id2.getId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/MockRecordWriter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/MockRecordWriter.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/MockRecordWriter.java
new file mode 100755
index 0000000..a40048e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/MockRecordWriter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+import java.util.ArrayList;
+
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.runtime.operators.DataSourceTask;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+
+public class MockRecordWriter extends RecordWriter<SerializationDelegate<StreamRecord<Tuple1<Integer>>>> {
+
+ public ArrayList<Integer> emittedRecords;
+
+ public MockRecordWriter(DataSourceTask<?> inputBase, Class<StreamRecord<Tuple1<Integer>>> outputClass) {
+ super(inputBase.getEnvironment().getWriter(0));
+ }
+
+ public boolean initList() {
+ emittedRecords = new ArrayList<Integer>();
+ return true;
+ }
+
+ @Override
+ public void emit(SerializationDelegate<StreamRecord<Tuple1<Integer>>> record) {
+ emittedRecords.add(record.getInstance().getObject().f0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/StreamVertexTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/StreamVertexTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/StreamVertexTest.java
new file mode 100644
index 0000000..b103d84
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/streamvertex/StreamVertexTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.streamvertex;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.co.CoMapFunction;
+import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class StreamVertexTest {
+
+ private static Map<Integer, Integer> data = new HashMap<Integer, Integer>();
+
+ public static class MySource implements SourceFunction<Tuple1<Integer>> {
+ private static final long serialVersionUID = 1L;
+
+ private Tuple1<Integer> tuple = new Tuple1<Integer>(0);
+
+ @Override
+ public void invoke(Collector<Tuple1<Integer>> collector) throws Exception {
+ for (int i = 0; i < 10; i++) {
+ tuple.f0 = i;
+ collector.collect(tuple);
+ }
+ }
+ }
+
+ public static class MyTask extends RichMapFunction<Tuple1<Integer>, Tuple2<Integer, Integer>> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<Integer, Integer> map(Tuple1<Integer> value) throws Exception {
+ Integer i = value.f0;
+ return new Tuple2<Integer, Integer>(i, i + 1);
+ }
+ }
+
+ public static class MySink implements SinkFunction<Tuple2<Integer, Integer>> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void invoke(Tuple2<Integer, Integer> tuple) {
+ Integer k = tuple.getField(0);
+ Integer v = tuple.getField(1);
+ data.put(k, v);
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private static final int PARALLELISM = 1;
+ private static final int SOURCE_PARALELISM = 1;
+ private static final long MEMORYSIZE = 32;
+
+ @Test
+ public void wrongJobGraph() {
+ LocalStreamEnvironment env = StreamExecutionEnvironment
+ .createLocalEnvironment(SOURCE_PARALELISM);
+
+
+ try {
+ env.fromCollection(null);
+ fail();
+ } catch (NullPointerException e) {
+ }
+
+ try {
+ env.fromElements();
+ fail();
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ env.generateSequence(-10, -30);
+ fail();
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ env.setBufferTimeout(-10);
+ fail();
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ env.generateSequence(1, 10).project(2);
+ fail();
+ } catch (RuntimeException e) {
+ }
+ }
+
+ private static class CoMap implements CoMapFunction<String, Long, String> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String map1(String value) {
+ return value;
+ }
+
+ @Override
+ public String map2(Long value) {
+ return value.toString();
+ }
+ }
+
+ static HashSet<String> resultSet;
+
+ private static class SetSink implements SinkFunction<String> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void invoke(String value) {
+ resultSet.add(value);
+ }
+ }
+
+ @Test
+ public void coTest() throws Exception {
+ StreamExecutionEnvironment env = new TestStreamEnvironment(SOURCE_PARALELISM, MEMORYSIZE);
+
+ DataStream<String> fromStringElements = env.fromElements("aa", "bb", "cc");
+ DataStream<Long> generatedSequence = env.generateSequence(0, 3);
+
+ fromStringElements.connect(generatedSequence).map(new CoMap()).addSink(new SetSink());
+
+ resultSet = new HashSet<String>();
+ env.execute();
+
+ HashSet<String> expectedSet = new HashSet<String>(Arrays.asList("aa", "bb", "cc", "0", "1",
+ "2", "3"));
+ assertEquals(expectedSet, resultSet);
+ }
+
+ @Test
+ public void runStream() throws Exception {
+ StreamExecutionEnvironment env = new TestStreamEnvironment(SOURCE_PARALELISM, MEMORYSIZE);
+
+ env.addSource(new MySource()).setParallelism(SOURCE_PARALELISM).map(new MyTask()).addSink(new MySink());
+
+ env.execute();
+ assertEquals(10, data.keySet().size());
+
+ for (Integer k : data.keySet()) {
+ assertEquals((Integer) (k + 1), data.get(k));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
new file mode 100644
index 0000000..e12b254
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class CosineDistanceTest {
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testCosineDistance() {
+
+ //Reference calculated using wolfram alpha
+ double[][][] testdata={
+ {{0,0,0},{0,0,0}},
+ {{0,0,0},{1,2,3}},
+ {{1,2,3},{0,0,0}},
+ {{1,2,3},{4,5,6}},
+ {{1,2,3},{-4,-5,-6}},
+ {{1,2,-3},{-4,5,-6}},
+ {{1,2,3,4},{5,6,7,8}},
+ {{1,2},{3,4}},
+ {{1},{2}},
+ };
+ double[] referenceSolutions={
+ 0,
+ 0,
+ 0,
+ 0.025368,
+ 1.974631,
+ 0.269026,
+ 0.031136,
+ 0.016130,
+ 0
+ };
+
+ for (int i = 0; i < testdata.length; i++) {
+ assertEquals("Wrong result for inputs " + arrayToString(testdata[i][0]) + " and "
+ + arrayToString(testdata[i][0]), referenceSolutions[i],
+ new CosineDistance().getDelta(testdata[i][0], testdata[i][1]), 0.000001);
+ }
+ }
+
+ private String arrayToString(double[] in){
+ if (in.length==0) return "{}";
+ String result="{";
+ for (double d:in){
+ result+=d+",";
+ }
+ return result.substring(0, result.length()-1)+"}";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
new file mode 100644
index 0000000..8c62497
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class EuclideanDistanceTest {
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testEuclideanDistance() {
+
+ //Reference calculated using wolfram alpha
+ double[][][] testdata={
+ {{0,0,0},{0,0,0}},
+ {{0,0,0},{1,2,3}},
+ {{1,2,3},{0,0,0}},
+ {{1,2,3},{4,5,6}},
+ {{1,2,3},{-4,-5,-6}},
+ {{1,2,-3},{-4,5,-6}},
+ {{1,2,3,4},{5,6,7,8}},
+ {{1,2},{3,4}},
+ {{1},{2}},
+ };
+ double[] referenceSolutions={
+ 0,
+ 3.741657,
+ 3.741657,
+ 5.196152,
+ 12.4499,
+ 6.557439,
+ 8.0,
+ 2.828427,
+ 1
+ };
+
+ for (int i = 0; i < testdata.length; i++) {
+ assertEquals("Wrong result for inputs " + arrayToString(testdata[i][0]) + " and "
+ + arrayToString(testdata[i][0]), referenceSolutions[i],
+ new EuclideanDistance().getDelta(testdata[i][0], testdata[i][1]), 0.000001);
+ }
+
+ }
+
+ private String arrayToString(double[] in){
+ if (in.length==0) return "{}";
+ String result="{";
+ for (double d:in){
+ result+=d+",";
+ }
+ return result.substring(0, result.length()-1)+"}";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java
new file mode 100644
index 0000000..17d3974
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple10;
+import org.apache.flink.api.java.tuple.Tuple11;
+import org.apache.flink.api.java.tuple.Tuple12;
+import org.apache.flink.api.java.tuple.Tuple13;
+import org.apache.flink.api.java.tuple.Tuple14;
+import org.apache.flink.api.java.tuple.Tuple15;
+import org.apache.flink.api.java.tuple.Tuple16;
+import org.apache.flink.api.java.tuple.Tuple17;
+import org.apache.flink.api.java.tuple.Tuple18;
+import org.apache.flink.api.java.tuple.Tuple19;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple20;
+import org.apache.flink.api.java.tuple.Tuple21;
+import org.apache.flink.api.java.tuple.Tuple22;
+import org.apache.flink.api.java.tuple.Tuple23;
+import org.apache.flink.api.java.tuple.Tuple24;
+import org.apache.flink.api.java.tuple.Tuple25;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple6;
+import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.tuple.Tuple9;
+import org.apache.flink.streaming.api.windowing.extractor.ArrayFromTuple;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ArrayFromTupleTest {
+
+ private String[] testStrings;
+
+ @Before
+ public void init() {
+ testStrings = new String[Tuple.MAX_ARITY];
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ testStrings[i] = Integer.toString(i);
+ }
+ }
+
+ @Test
+ public void testConvertFromTupleToArray() throws InstantiationException, IllegalAccessException {
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ Tuple currentTuple = (Tuple) CLASSES[i].newInstance();
+ String[] currentArray = new String[i + 1];
+ for (int j = 0; j <= i; j++) {
+ currentTuple.setField(testStrings[j], j);
+ currentArray[j] = testStrings[j];
+ }
+ arrayEqualityCheck(currentArray, new ArrayFromTuple().extract(currentTuple));
+ }
+ }
+
+ @Test
+ public void testUserSpecifiedOrder() throws InstantiationException, IllegalAccessException {
+ Tuple currentTuple = (Tuple) CLASSES[Tuple.MAX_ARITY - 1].newInstance();
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ currentTuple.setField(testStrings[i], i);
+ }
+
+ String[] expected = { testStrings[5], testStrings[3], testStrings[6], testStrings[7],
+ testStrings[0] };
+ arrayEqualityCheck(expected, new ArrayFromTuple(5, 3, 6, 7, 0).extract(currentTuple));
+
+ String[] expected2 = { testStrings[0], testStrings[Tuple.MAX_ARITY - 1] };
+ arrayEqualityCheck(expected2,
+ new ArrayFromTuple(0, Tuple.MAX_ARITY - 1).extract(currentTuple));
+
+ String[] expected3 = { testStrings[Tuple.MAX_ARITY - 1], testStrings[0] };
+ arrayEqualityCheck(expected3,
+ new ArrayFromTuple(Tuple.MAX_ARITY - 1, 0).extract(currentTuple));
+
+ String[] expected4 = { testStrings[13], testStrings[4], testStrings[5], testStrings[4],
+ testStrings[2], testStrings[8], testStrings[6], testStrings[2], testStrings[8],
+ testStrings[3], testStrings[5], testStrings[2], testStrings[16], testStrings[4],
+ testStrings[3], testStrings[2], testStrings[6], testStrings[4], testStrings[7],
+ testStrings[4], testStrings[2], testStrings[8], testStrings[7], testStrings[2] };
+ arrayEqualityCheck(expected4, new ArrayFromTuple(13, 4, 5, 4, 2, 8, 6, 2, 8, 3, 5, 2, 16,
+ 4, 3, 2, 6, 4, 7, 4, 2, 8, 7, 2).extract(currentTuple));
+ }
+
+ private void arrayEqualityCheck(Object[] array1, Object[] array2) {
+ assertEquals("The result arrays must have the same length", array1.length, array2.length);
+ for (int i = 0; i < array1.length; i++) {
+ assertEquals("Unequal fields at position " + i, array1[i], array2[i]);
+ }
+ }
+
+ private static final Class<?>[] CLASSES = new Class<?>[] { Tuple1.class, Tuple2.class,
+ Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
+ Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
+ Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
+ Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
+ Tuple24.class, Tuple25.class };
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java
new file mode 100644
index 0000000..82a876a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.windowing.extractor.ArrayFromTuple;
+import org.apache.flink.streaming.api.windowing.extractor.ConcatinatedExtract;
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.extractor.FieldFromArray;
+import org.apache.flink.streaming.api.windowing.extractor.FieldFromTuple;
+import org.apache.flink.streaming.api.windowing.extractor.FieldsFromArray;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ConcatinatedExtractTest {
+
+ private String[] testStringArray1 = { "1", "2", "3" };
+ private int[] testIntArray1 = { 1, 2, 3 };
+ private String[] testStringArray2 = { "4", "5", "6" };
+ private int[] testIntArray2 = { 4, 5, 6 };
+ private String[] testStringArray3 = { "7", "8", "9" };
+ private int[] testIntArray3 = { 7, 8, 9 };
+ private Tuple2<String[], int[]>[] testTuple2Array;
+ private Tuple2<String[], int[]> testTuple2;
+ private Tuple2<Tuple2<String[], int[]>, Tuple2<String[], int[]>[]> testData;
+
+ @SuppressWarnings("unchecked")
+ @Before
+ public void setupData() {
+ testTuple2Array = new Tuple2[2];
+ testTuple2Array[0] = new Tuple2<String[], int[]>(testStringArray1, testIntArray2);
+ testTuple2Array[1] = new Tuple2<String[], int[]>(testStringArray2, testIntArray1);
+
+ testTuple2 = new Tuple2<String[], int[]>(testStringArray3, testIntArray3);
+
+ testData = new Tuple2<Tuple2<String[], int[]>, Tuple2<String[], int[]>[]>(testTuple2,
+ testTuple2Array);
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void test1() {
+ Extractor ext = new ConcatinatedExtract(new FieldFromTuple(0), new FieldFromTuple(1))
+ .add(new FieldsFromArray(Integer.class, 2, 1, 0));
+ int[] expected = { testIntArray3[2], testIntArray3[1], testIntArray3[0] };
+ assertEquals(new Integer(expected[0]), ((Integer[]) ext.extract(testData))[0]);
+ assertEquals(new Integer(expected[1]), ((Integer[]) ext.extract(testData))[1]);
+ assertEquals(new Integer(expected[2]), ((Integer[]) ext.extract(testData))[2]);
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Test
+ public void test2() {
+ Extractor ext = new ConcatinatedExtract(new FieldFromTuple(1), // Tuple2<String[],int[]>[]
+ new FieldsFromArray(Tuple2.class, 1)) // Tuple2<String[],int[]>[]
+ .add(new FieldFromArray(0)) // Tuple2<String[],int[]>
+ .add(new ArrayFromTuple(0)) // Object[] (Containing String[])
+ .add(new FieldFromArray(0)) // String[]
+ .add(new FieldFromArray(1)); // String
+
+ String expected2 = testStringArray2[1];
+ assertEquals(expected2, ext.extract(testData));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java
new file mode 100644
index 0000000..2d4dbcf
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.streaming.api.windowing.extractor.FieldFromArray;
+import org.junit.Test;
+
+public class FieldFromArrayTest {
+
+ String[] testStringArray = { "0", "1", "2", "3", "4" };
+ Integer[] testIntegerArray = { 10, 11, 12, 13, 14 };
+ int[] testIntArray = { 20, 21, 22, 23, 24 };
+
+ @Test
+ public void testStringArray() {
+ for (int i = 0; i < this.testStringArray.length; i++) {
+ assertEquals(this.testStringArray[i],
+ new FieldFromArray<String>(i).extract(testStringArray));
+ }
+ }
+
+ @Test
+ public void testIntegerArray() {
+ for (int i = 0; i < this.testIntegerArray.length; i++) {
+ assertEquals(this.testIntegerArray[i],
+ new FieldFromArray<String>(i).extract(testIntegerArray));
+ }
+ }
+
+ @Test
+ public void testIntArray() {
+ for (int i = 0; i < this.testIntArray.length; i++) {
+ assertEquals(new Integer(this.testIntArray[i]),
+ new FieldFromArray<Integer>(i).extract(testIntArray));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java
new file mode 100644
index 0000000..528611a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple10;
+import org.apache.flink.api.java.tuple.Tuple11;
+import org.apache.flink.api.java.tuple.Tuple12;
+import org.apache.flink.api.java.tuple.Tuple13;
+import org.apache.flink.api.java.tuple.Tuple14;
+import org.apache.flink.api.java.tuple.Tuple15;
+import org.apache.flink.api.java.tuple.Tuple16;
+import org.apache.flink.api.java.tuple.Tuple17;
+import org.apache.flink.api.java.tuple.Tuple18;
+import org.apache.flink.api.java.tuple.Tuple19;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple20;
+import org.apache.flink.api.java.tuple.Tuple21;
+import org.apache.flink.api.java.tuple.Tuple22;
+import org.apache.flink.api.java.tuple.Tuple23;
+import org.apache.flink.api.java.tuple.Tuple24;
+import org.apache.flink.api.java.tuple.Tuple25;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple6;
+import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.tuple.Tuple9;
+import org.apache.flink.streaming.api.windowing.extractor.FieldFromTuple;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FieldFromTupleTest {
+
+ private String[] testStrings;
+
+ @Before
+ public void init() {
+ testStrings = new String[Tuple.MAX_ARITY];
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ testStrings[i] = Integer.toString(i);
+ }
+ }
+
+ @Test
+ public void testSingleFieldExtraction() throws InstantiationException, IllegalAccessException {
+ // extract single fields
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ Tuple current = (Tuple) CLASSES[i].newInstance();
+ for (int j = 0; j < i; j++) {
+ current.setField(testStrings[j], j);
+ }
+ for (int j = 0; j < i; j++) {
+ assertEquals(testStrings[j], new FieldFromTuple<String>(j).extract(current));
+ }
+ }
+ }
+
+ private static final Class<?>[] CLASSES = new Class<?>[] { Tuple1.class, Tuple2.class,
+ Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
+ Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
+ Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
+ Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
+ Tuple24.class, Tuple25.class };
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java
new file mode 100644
index 0000000..3139aa5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.flink.streaming.api.windowing.extractor.FieldsFromArray;
+import org.junit.Test;
+
+public class FieldsFromArrayTest {
+
+ String[] testStringArray = { "0", "1", "2", "3", "4" };
+ Integer[] testIntegerArray = { 10, 11, 12, 13, 14 };
+ int[] testIntArray = { 20, 21, 22, 23, 24 };
+
+ @Test
+ public void testStringArray() {
+ // check single field extraction
+ for (int i = 0; i < testStringArray.length; i++) {
+ String[] tmp = { testStringArray[i] };
+ arrayEqualityCheck(tmp,
+ new FieldsFromArray<String>(String.class, i).extract(testStringArray));
+ }
+
+ // check reverse order
+ String[] reverseOrder = new String[testStringArray.length];
+ for (int i = 0; i < testStringArray.length; i++) {
+ reverseOrder[i] = testStringArray[testStringArray.length - i - 1];
+ }
+ arrayEqualityCheck(reverseOrder,
+ new FieldsFromArray<String>(String.class, 4, 3, 2, 1, 0).extract(testStringArray));
+
+ // check picking fields and reorder
+ String[] crazyOrder = { testStringArray[4], testStringArray[1], testStringArray[2] };
+ arrayEqualityCheck(crazyOrder,
+ new FieldsFromArray<String>(String.class, 4, 1, 2).extract(testStringArray));
+ }
+
+ @Test
+ public void testIntegerArray() {
+ // check single field extraction
+ for (int i = 0; i < testIntegerArray.length; i++) {
+ Integer[] tmp = { testIntegerArray[i] };
+ arrayEqualityCheck(tmp,
+ new FieldsFromArray<Integer>(Integer.class, i).extract(testIntegerArray));
+ }
+
+ // check reverse order
+ Integer[] reverseOrder = new Integer[testIntegerArray.length];
+ for (int i = 0; i < testIntegerArray.length; i++) {
+ reverseOrder[i] = testIntegerArray[testIntegerArray.length - i - 1];
+ }
+ arrayEqualityCheck(reverseOrder,
+ new FieldsFromArray<Integer>(Integer.class, 4, 3, 2, 1, 0)
+ .extract(testIntegerArray));
+
+ // check picking fields and reorder
+ Integer[] crazyOrder = { testIntegerArray[4], testIntegerArray[1], testIntegerArray[2] };
+ arrayEqualityCheck(crazyOrder,
+ new FieldsFromArray<Integer>(Integer.class, 4, 1, 2).extract(testIntegerArray));
+
+ }
+
+ @Test
+ public void testIntArray() {
+ for (int i = 0; i < testIntArray.length; i++) {
+ Integer[] tmp = { testIntArray[i] };
+ arrayEqualityCheck(tmp,
+ new FieldsFromArray<Integer>(Integer.class, i).extract(testIntArray));
+ }
+
+ // check reverse order
+ Integer[] reverseOrder = new Integer[testIntArray.length];
+ for (int i = 0; i < testIntArray.length; i++) {
+ reverseOrder[i] = testIntArray[testIntArray.length - i - 1];
+ }
+ arrayEqualityCheck(reverseOrder,
+ new FieldsFromArray<Integer>(Integer.class, 4, 3, 2, 1, 0).extract(testIntArray));
+
+ // check picking fields and reorder
+ Integer[] crazyOrder = { testIntArray[4], testIntArray[1], testIntArray[2] };
+ arrayEqualityCheck(crazyOrder,
+ new FieldsFromArray<Integer>(Integer.class, 4, 1, 2).extract(testIntArray));
+
+ }
+
+ private void arrayEqualityCheck(Object[] array1, Object[] array2) {
+ assertEquals("The result arrays must have the same length", array1.length, array2.length);
+ for (int i = 0; i < array1.length; i++) {
+ assertEquals("Unequal fields at position " + i, array1[i], array2[i]);
+ }
+ }
+}
[20/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/OutgoingEdge.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/OutgoingEdge.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/OutgoingEdge.java
new file mode 100644
index 0000000..6d97525
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/OutgoingEdge.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java;
+
+/**
+ * <tt>Edge</tt> objects represent edges between vertices. Edges are defined by their source and target
+ * vertex id. Edges may have an associated value (for example a weight or a distance), if the
+ * graph algorithm was initialized with the
+ * {@link VertexCentricIteration#withValuedEdges(org.apache.flink.api.java.DataSet, VertexUpdateFunction, MessagingFunction, int)}
+ * method.
+ *
+ * @param <VertexKey> The type of the vertex key.
+ * @param <EdgeValue> The type of the value associated with the edge. For scenarios where the edges do not hold
+ * value, this type may be arbitrary.
+ */
+public final class OutgoingEdge<VertexKey extends Comparable<VertexKey>, EdgeValue> implements java.io.Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private VertexKey target;
+
+ private EdgeValue edgeValue;
+
+ void set(VertexKey target, EdgeValue edgeValue) {
+ this.target = target;
+ this.edgeValue = edgeValue;
+ }
+
+ /**
+ * Gets the target vertex id.
+ *
+ * @return The target vertex id.
+ */
+ public VertexKey target() {
+ return target;
+ }
+
+ /**
+ * Gets the value associated with the edge. The value may be null if the iteration was initialized with
+ * an edge data set without edge values.
+ * Typical examples of edge values are weights or distances of the path represented by the edge.
+ *
+ * @return The value associated with the edge.
+ */
+ public EdgeValue edgeValue() {
+ return edgeValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
new file mode 100644
index 0000000..4f84467
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
@@ -0,0 +1,599 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.Validate;
+import org.apache.flink.api.common.aggregators.Aggregator;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.operators.DeltaIteration;
+import org.apache.flink.api.common.functions.RichCoGroupFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.operators.CoGroupOperator;
+import org.apache.flink.api.java.operators.CustomUnaryOperation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.Collector;
+
+/**
+ * This class represents iterative graph computations, programmed in a vertex-centric perspective.
+ * It is a special case of <i>Bulk Synchronous Parallel<i> computation. The paradigm has also been
+ * implemented by Google's <i>Pregel</i> system and by <i>Apache Giraph</i>.
+ * <p>
+ * Vertex centric algorithms operate on graphs, which are defined through vertices and edges. The
+ * algorithms send messages along the edges and update the state of vertices based on
+ * the old state and the incoming messages. All vertices have an initial state.
+ * The computation terminates once no vertex updates it state any more.
+ * Additionally, a maximum number of iterations (supersteps) may be specified.
+ * <p>
+ * The computation is here represented by two functions:
+ * <ul>
+ * <li>The {@link VertexUpdateFunction} receives incoming messages and may updates the state for
+ * the vertex. If a state is updated, messages are sent from this vertex. Initially, all vertices are
+ * considered updated.</li>
+ * <li>The {@link MessagingFunction} takes the new vertex state and sends messages along the outgoing
+ * edges of the vertex. The outgoing edges may optionally have an associated value, such as a weight.</li>
+ * </ul>
+ * <p>
+ * Vertex-centric graph iterations are instantiated by the
+ * {@link #withPlainEdges(DataSet, VertexUpdateFunction, MessagingFunction, int)} method, or the
+ * {@link #withValuedEdges(DataSet, VertexUpdateFunction, MessagingFunction, int)} method, depending on whether
+ * the graph's edges are carrying values.
+ *
+ * @param <VertexKey> The type of the vertex key (the vertex identifier).
+ * @param <VertexValue> The type of the vertex value (the state of the vertex).
+ * @param <Message> The type of the message sent between vertices along the edges.
+ * @param <EdgeValue> The type of the values that are associated with the edges.
+ */
+public class VertexCentricIteration<VertexKey extends Comparable<VertexKey>, VertexValue, Message, EdgeValue>
+ implements CustomUnaryOperation<Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, VertexValue>>
+{
+ private final VertexUpdateFunction<VertexKey, VertexValue, Message> updateFunction;
+
+ private final MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> messagingFunction;
+
+ private final DataSet<Tuple2<VertexKey, VertexKey>> edgesWithoutValue;
+
+ private final DataSet<Tuple3<VertexKey, VertexKey, EdgeValue>> edgesWithValue;
+
+ private final Map<String, Aggregator<?>> aggregators;
+
+ private final int maximumNumberOfIterations;
+
+ private final List<Tuple2<String, DataSet<?>>> bcVarsUpdate = new ArrayList<Tuple2<String,DataSet<?>>>(4);
+
+ private final List<Tuple2<String, DataSet<?>>> bcVarsMessaging = new ArrayList<Tuple2<String,DataSet<?>>>(4);
+
+ private final TypeInformation<Message> messageType;
+
+ private DataSet<Tuple2<VertexKey, VertexValue>> initialVertices;
+
+ private String name;
+
+ private int parallelism = -1;
+
+ private boolean unmanagedSolutionSet;
+
+ // ----------------------------------------------------------------------------------
+
+ private VertexCentricIteration(VertexUpdateFunction<VertexKey, VertexValue, Message> uf,
+ MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf,
+ DataSet<Tuple2<VertexKey, VertexKey>> edgesWithoutValue,
+ int maximumNumberOfIterations)
+ {
+ Validate.notNull(uf);
+ Validate.notNull(mf);
+ Validate.notNull(edgesWithoutValue);
+ Validate.isTrue(maximumNumberOfIterations > 0, "The maximum number of iterations must be at least one.");
+
+ // check that the edges are actually a valid tuple set of vertex key types
+ TypeInformation<Tuple2<VertexKey, VertexKey>> edgesType = edgesWithoutValue.getType();
+ Validate.isTrue(edgesType.isTupleType() && edgesType.getArity() == 2, "The edges data set (for edges without edge values) must consist of 2-tuples.");
+
+ TupleTypeInfo<?> tupleInfo = (TupleTypeInfo<?>) edgesType;
+ Validate.isTrue(tupleInfo.getTypeAt(0).equals(tupleInfo.getTypeAt(1))
+ && Comparable.class.isAssignableFrom(tupleInfo.getTypeAt(0).getTypeClass()),
+ "Both tuple fields (source and target vertex id) must be of the data type that represents the vertex key and implement the java.lang.Comparable interface.");
+
+ this.updateFunction = uf;
+ this.messagingFunction = mf;
+ this.edgesWithoutValue = edgesWithoutValue;
+ this.edgesWithValue = null;
+ this.maximumNumberOfIterations = maximumNumberOfIterations;
+ this.aggregators = new HashMap<String, Aggregator<?>>();
+
+ this.messageType = getMessageType(mf);
+ }
+
+ private VertexCentricIteration(VertexUpdateFunction<VertexKey, VertexValue, Message> uf,
+ MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf,
+ DataSet<Tuple3<VertexKey, VertexKey, EdgeValue>> edgesWithValue,
+ int maximumNumberOfIterations,
+ boolean edgeHasValueMarker)
+ {
+ Validate.notNull(uf);
+ Validate.notNull(mf);
+ Validate.notNull(edgesWithValue);
+ Validate.isTrue(maximumNumberOfIterations > 0, "The maximum number of iterations must be at least one.");
+
+ // check that the edges are actually a valid tuple set of vertex key types
+ TypeInformation<Tuple3<VertexKey, VertexKey, EdgeValue>> edgesType = edgesWithValue.getType();
+ Validate.isTrue(edgesType.isTupleType() && edgesType.getArity() == 3, "The edges data set (for edges with edge values) must consist of 3-tuples.");
+
+ TupleTypeInfo<?> tupleInfo = (TupleTypeInfo<?>) edgesType;
+ Validate.isTrue(tupleInfo.getTypeAt(0).equals(tupleInfo.getTypeAt(1))
+ && Comparable.class.isAssignableFrom(tupleInfo.getTypeAt(0).getTypeClass()),
+ "The first two tuple fields (source and target vertex id) must be of the data type that represents the vertex key and implement the java.lang.Comparable interface.");
+
+ Validate.isTrue(maximumNumberOfIterations > 0, "The maximum number of iterations must be at least one.");
+
+ this.updateFunction = uf;
+ this.messagingFunction = mf;
+ this.edgesWithoutValue = null;
+ this.edgesWithValue = edgesWithValue;
+ this.maximumNumberOfIterations = maximumNumberOfIterations;
+ this.aggregators = new HashMap<String, Aggregator<?>>();
+
+ this.messageType = getMessageType(mf);
+ }
+
+ private TypeInformation<Message> getMessageType(MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf) {
+ return TypeExtractor.createTypeInfo(MessagingFunction.class, mf.getClass(), 2, null, null);
+ }
+
+ /**
+ * Registers a new aggregator. Aggregators registered here are available during the execution of the vertex updates
+ * via {@link VertexUpdateFunction#getIterationAggregator(String)} and
+ * {@link VertexUpdateFunction#getPreviousIterationAggregate(String)}.
+ *
+ * @param name The name of the aggregator, used to retrieve it and its aggregates during execution.
+ * @param aggregator The aggregator.
+ */
+ public void registerAggregator(String name, Aggregator<?> aggregator) {
+ this.aggregators.put(name, aggregator);
+ }
+
+ /**
+ * Adds a data set as a broadcast set to the messaging function.
+ *
+ * @param name The name under which the broadcast data is available in the messaging function.
+ * @param data The data set to be broadcasted.
+ */
+ public void addBroadcastSetForMessagingFunction(String name, DataSet<?> data) {
+ this.bcVarsMessaging.add(new Tuple2<String, DataSet<?>>(name, data));
+ }
+
+ /**
+ * Adds a data set as a broadcast set to the vertex update function.
+ *
+ * @param name The name under which the broadcast data is available in the vertex update function.
+ * @param data The data set to be broadcasted.
+ */
+ public void addBroadcastSetForUpdateFunction(String name, DataSet<?> data) {
+ this.bcVarsUpdate.add(new Tuple2<String, DataSet<?>>(name, data));
+ }
+
+ /**
+ * Sets the name for the vertex-centric iteration. The name is displayed in logs and messages.
+ *
+ * @param name The name for the iteration.
+ */
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Gets the name from this vertex-centric iteration.
+ *
+ * @return The name of the iteration.
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Sets the degree of parallelism for the iteration.
+ *
+ * @param parallelism The degree of parallelism.
+ */
+ public void setParallelism(int parallelism) {
+ Validate.isTrue(parallelism > 0 || parallelism == -1, "The degree of parallelism must be positive, or -1 (use default).");
+ this.parallelism = parallelism;
+ }
+
+ /**
+ * Gets the iteration's degree of parallelism.
+ *
+ * @return The iterations parallelism, or -1, if not set.
+ */
+ public int getParallelism() {
+ return parallelism;
+ }
+
+ /**
+ * Defines whether the solution set is kept in managed memory (Flink's internal way of keeping object
+ * in serialized form) or as a simple object map.
+ * By default, the solution set runs in managed memory.
+ *
+ * @param unmanaged True, to keep the solution set in unmanaged memory, false otherwise.
+ */
+ public void setSolutionSetUnmanagedMemory(boolean unmanaged) {
+ this.unmanagedSolutionSet = unmanaged;
+ }
+
+ /**
+ * Gets whether the solution set is kept in managed memory (Flink's internal way of keeping object
+ * in serialized form) or as a simple object map.
+ * By default, the solution set runs in managed memory.
+ *
+ * @return True, if the solution set is in unmanaged memory, false otherwise.
+ */
+ public boolean isSolutionSetUnmanagedMemory() {
+ return this.unmanagedSolutionSet;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Custom Operator behavior
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Sets the input data set for this operator. In the case of this operator this input data set represents
+ * the set of vertices with their initial state.
+ *
+ * @param inputData The input data set, which in the case of this operator represents the set of
+ * vertices with their initial state.
+ *
+ * @see org.apache.flink.api.java.operators.CustomUnaryOperation#setInput(org.apache.flink.api.java.DataSet)
+ */
+ @Override
+ public void setInput(DataSet<Tuple2<VertexKey, VertexValue>> inputData) {
+ // sanity check that we really have two tuples
+ TypeInformation<Tuple2<VertexKey, VertexValue>> inputType = inputData.getType();
+ Validate.isTrue(inputType.isTupleType() && inputType.getArity() == 2, "The input data set (the initial vertices) must consist of 2-tuples.");
+
+ // check that the key type here is the same as for the edges
+ TypeInformation<VertexKey> keyType = ((TupleTypeInfo<?>) inputType).getTypeAt(0);
+ TypeInformation<?> edgeType = edgesWithoutValue != null ? edgesWithoutValue.getType() : edgesWithValue.getType();
+ TypeInformation<VertexKey> edgeKeyType = ((TupleTypeInfo<?>) edgeType).getTypeAt(0);
+
+ Validate.isTrue(keyType.equals(edgeKeyType), "The first tuple field (the vertex id) of the input data set (the initial vertices) " +
+ "must be the same data type as the first fields of the edge data set (the source vertex id). " +
+ "Here, the key type for the vertex ids is '%s' and the key type for the edges is '%s'.", keyType, edgeKeyType);
+
+ this.initialVertices = inputData;
+ }
+
+ /**
+ * Creates the operator that represents this vertex-centric graph computation.
+ *
+ * @return The operator that represents this vertex-centric graph computation.
+ */
+ @Override
+ public DataSet<Tuple2<VertexKey, VertexValue>> createResult() {
+ if (this.initialVertices == null) {
+ throw new IllegalStateException("The input data set has not been set.");
+ }
+
+ // prepare some type information
+ TypeInformation<Tuple2<VertexKey, VertexValue>> vertexTypes = initialVertices.getType();
+ TypeInformation<VertexKey> keyType = ((TupleTypeInfo<?>) initialVertices.getType()).getTypeAt(0);
+ TypeInformation<Tuple2<VertexKey, Message>> messageTypeInfo = new TupleTypeInfo<Tuple2<VertexKey,Message>>(keyType, messageType);
+
+ // set up the iteration operator
+ final String name = (this.name != null) ? this.name :
+ "Vertex-centric iteration (" + updateFunction + " | " + messagingFunction + ")";
+ final int[] zeroKeyPos = new int[] {0};
+
+ final DeltaIteration<Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, VertexValue>> iteration =
+ this.initialVertices.iterateDelta(this.initialVertices, this.maximumNumberOfIterations, zeroKeyPos);
+ iteration.name(name);
+ iteration.parallelism(parallelism);
+ iteration.setSolutionSetUnManaged(unmanagedSolutionSet);
+
+ // register all aggregators
+ for (Map.Entry<String, Aggregator<?>> entry : this.aggregators.entrySet()) {
+ iteration.registerAggregator(entry.getKey(), entry.getValue());
+ }
+
+ // build the messaging function (co group)
+ CoGroupOperator<?, ?, Tuple2<VertexKey, Message>> messages;
+ if (edgesWithoutValue != null) {
+ MessagingUdfNoEdgeValues<VertexKey, VertexValue, Message> messenger = new MessagingUdfNoEdgeValues<VertexKey, VertexValue, Message>(messagingFunction, messageTypeInfo);
+ messages = this.edgesWithoutValue.coGroup(iteration.getWorkset()).where(0).equalTo(0).with(messenger);
+ }
+ else {
+ MessagingUdfWithEdgeValues<VertexKey, VertexValue, Message, EdgeValue> messenger = new MessagingUdfWithEdgeValues<VertexKey, VertexValue, Message, EdgeValue>(messagingFunction, messageTypeInfo);
+ messages = this.edgesWithValue.coGroup(iteration.getWorkset()).where(0).equalTo(0).with(messenger);
+ }
+
+ // configure coGroup message function with name and broadcast variables
+ messages = messages.name("Messaging");
+ for (Tuple2<String, DataSet<?>> e : this.bcVarsMessaging) {
+ messages = messages.withBroadcastSet(e.f1, e.f0);
+ }
+
+ VertexUpdateUdf<VertexKey, VertexValue, Message> updateUdf = new VertexUpdateUdf<VertexKey, VertexValue, Message>(updateFunction, vertexTypes);
+
+ // build the update function (co group)
+ CoGroupOperator<?, ?, Tuple2<VertexKey, VertexValue>> updates =
+ messages.coGroup(iteration.getSolutionSet()).where(0).equalTo(0).with(updateUdf);
+
+ // configure coGroup update function with name and broadcast variables
+ updates = updates.name("Vertex State Updates");
+ for (Tuple2<String, DataSet<?>> e : this.bcVarsUpdate) {
+ updates = updates.withBroadcastSet(e.f1, e.f0);
+ }
+
+ // let the operator know that we preserve the key field
+ updates.withForwardedFieldsFirst("0").withForwardedFieldsSecond("0");
+
+ return iteration.closeWith(updates, updates);
+
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Constructor builders to avoid signature conflicts with generic type erasure
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Creates a new vertex-centric iteration operator for graphs where the edges are not associated with a value.
+ *
+ * @param edgesWithoutValue The data set containing edges. Edges are represented as 2-tuples: (source-id, target-id)
+ * @param vertexUpdateFunction The function that updates the state of the vertices from the incoming messages.
+ * @param messagingFunction The function that turns changed vertex states into messages along the edges.
+ *
+ * @param <VertexKey> The type of the vertex key (the vertex identifier).
+ * @param <VertexValue> The type of the vertex value (the state of the vertex).
+ * @param <Message> The type of the message sent between vertices along the edges.
+ *
+ * @return An in stance of the vertex-centric graph computation operator.
+ */
+ public static final <VertexKey extends Comparable<VertexKey>, VertexValue, Message>
+ VertexCentricIteration<VertexKey, VertexValue, Message, ?> withPlainEdges(
+ DataSet<Tuple2<VertexKey, VertexKey>> edgesWithoutValue,
+ VertexUpdateFunction<VertexKey, VertexValue, Message> vertexUpdateFunction,
+ MessagingFunction<VertexKey, VertexValue, Message, ?> messagingFunction,
+ int maximumNumberOfIterations)
+ {
+ @SuppressWarnings("unchecked")
+ MessagingFunction<VertexKey, VertexValue, Message, Object> tmf =
+ (MessagingFunction<VertexKey, VertexValue, Message, Object>) messagingFunction;
+
+ return new VertexCentricIteration<VertexKey, VertexValue, Message, Object>(vertexUpdateFunction, tmf, edgesWithoutValue, maximumNumberOfIterations);
+ }
+
+ /**
+ * Creates a new vertex-centric iteration operator for graphs where the edges are associated with a value (such as
+ * a weight or distance).
+ *
+ * @param edgesWithValue The data set containing edges. Edges are represented as 2-tuples: (source-id, target-id)
+ * @param uf The function that updates the state of the vertices from the incoming messages.
+ * @param mf The function that turns changed vertex states into messages along the edges.
+ *
+ * @param <VertexKey> The type of the vertex key (the vertex identifier).
+ * @param <VertexValue> The type of the vertex value (the state of the vertex).
+ * @param <Message> The type of the message sent between vertices along the edges.
+ * @param <EdgeValue> The type of the values that are associated with the edges.
+ *
+ * @return An in stance of the vertex-centric graph computation operator.
+ */
+ public static final <VertexKey extends Comparable<VertexKey>, VertexValue, Message, EdgeValue>
+ VertexCentricIteration<VertexKey, VertexValue, Message, EdgeValue> withValuedEdges(
+ DataSet<Tuple3<VertexKey, VertexKey, EdgeValue>> edgesWithValue,
+ VertexUpdateFunction<VertexKey, VertexValue, Message> uf,
+ MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf,
+ int maximumNumberOfIterations)
+ {
+ return new VertexCentricIteration<VertexKey, VertexValue, Message, EdgeValue>(uf, mf, edgesWithValue, maximumNumberOfIterations, true);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Wrapping UDFs
+ // --------------------------------------------------------------------------------------------
+
+ private static final class VertexUpdateUdf<VertexKey extends Comparable<VertexKey>, VertexValue, Message>
+ extends RichCoGroupFunction<Tuple2<VertexKey, Message>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, VertexValue>>
+ implements ResultTypeQueryable<Tuple2<VertexKey, VertexValue>>
+ {
+ private static final long serialVersionUID = 1L;
+
+ private final VertexUpdateFunction<VertexKey, VertexValue, Message> vertexUpdateFunction;
+
+ private final MessageIterator<Message> messageIter = new MessageIterator<Message>();
+
+ private transient TypeInformation<Tuple2<VertexKey, VertexValue>> resultType;
+
+
+ private VertexUpdateUdf(VertexUpdateFunction<VertexKey, VertexValue, Message> vertexUpdateFunction,
+ TypeInformation<Tuple2<VertexKey, VertexValue>> resultType)
+ {
+ this.vertexUpdateFunction = vertexUpdateFunction;
+ this.resultType = resultType;
+ }
+
+ @Override
+ public void coGroup(Iterable<Tuple2<VertexKey, Message>> messages, Iterable<Tuple2<VertexKey, VertexValue>> vertex,
+ Collector<Tuple2<VertexKey, VertexValue>> out)
+ throws Exception
+ {
+ final Iterator<Tuple2<VertexKey, VertexValue>> vertexIter = vertex.iterator();
+
+ if (vertexIter.hasNext()) {
+ Tuple2<VertexKey, VertexValue> vertexState = vertexIter.next();
+
+ @SuppressWarnings("unchecked")
+ Iterator<Tuple2<?, Message>> downcastIter = (Iterator<Tuple2<?, Message>>) (Iterator<?>) messages.iterator();
+ messageIter.setSource(downcastIter);
+
+ vertexUpdateFunction.setOutput(vertexState, out);
+ vertexUpdateFunction.updateVertex(vertexState.f0, vertexState.f1, messageIter);
+ }
+ else {
+ final Iterator<Tuple2<VertexKey, Message>> messageIter = messages.iterator();
+ if (messageIter.hasNext()) {
+ String message = "Target vertex does not exist!.";
+ try {
+ Tuple2<VertexKey, Message> next = messageIter.next();
+ message = "Target vertex '" + next.f0 + "' does not exist!.";
+ } catch (Throwable t) {}
+ throw new Exception(message);
+ } else {
+ throw new Exception();
+ }
+ }
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ if (getIterationRuntimeContext().getSuperstepNumber() == 1) {
+ this.vertexUpdateFunction.init(getIterationRuntimeContext());
+ }
+ this.vertexUpdateFunction.preSuperstep();
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.vertexUpdateFunction.postSuperstep();
+ }
+
+ @Override
+ public TypeInformation<Tuple2<VertexKey, VertexValue>> getProducedType() {
+ return this.resultType;
+ }
+ }
+
+ /*
+ * UDF that encapsulates the message sending function for graphs where the edges have no associated values.
+ */
+ private static final class MessagingUdfNoEdgeValues<VertexKey extends Comparable<VertexKey>, VertexValue, Message>
+ extends RichCoGroupFunction<Tuple2<VertexKey, VertexKey>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, Message>>
+ implements ResultTypeQueryable<Tuple2<VertexKey, Message>>
+ {
+ private static final long serialVersionUID = 1L;
+
+ private final MessagingFunction<VertexKey, VertexValue, Message, ?> messagingFunction;
+
+ private transient TypeInformation<Tuple2<VertexKey, Message>> resultType;
+
+
+ private MessagingUdfNoEdgeValues(MessagingFunction<VertexKey, VertexValue, Message, ?> messagingFunction,
+ TypeInformation<Tuple2<VertexKey, Message>> resultType)
+ {
+ this.messagingFunction = messagingFunction;
+ this.resultType = resultType;
+ }
+
+ @Override
+ public void coGroup(Iterable<Tuple2<VertexKey, VertexKey>> edges,
+ Iterable<Tuple2<VertexKey, VertexValue>> state, Collector<Tuple2<VertexKey, Message>> out)
+ throws Exception
+ {
+ final Iterator<Tuple2<VertexKey, VertexValue>> stateIter = state.iterator();
+
+ if (stateIter.hasNext()) {
+ Tuple2<VertexKey, VertexValue> newVertexState = stateIter.next();
+ messagingFunction.set((Iterator<?>) edges.iterator(), out);
+ messagingFunction.sendMessages(newVertexState.f0, newVertexState.f1);
+ }
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ if (getIterationRuntimeContext().getSuperstepNumber() == 1) {
+ this.messagingFunction.init(getIterationRuntimeContext(), false);
+ }
+
+ this.messagingFunction.preSuperstep();
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.messagingFunction.postSuperstep();
+ }
+
+ @Override
+ public TypeInformation<Tuple2<VertexKey, Message>> getProducedType() {
+ return this.resultType;
+ }
+ }
+
+ /*
+ * UDF that encapsulates the message sending function for graphs where the edges have an associated value.
+ */
+ private static final class MessagingUdfWithEdgeValues<VertexKey extends Comparable<VertexKey>, VertexValue, Message, EdgeValue>
+ extends RichCoGroupFunction<Tuple3<VertexKey, VertexKey, EdgeValue>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, Message>>
+ implements ResultTypeQueryable<Tuple2<VertexKey, Message>>
+ {
+ private static final long serialVersionUID = 1L;
+
+ private final MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> messagingFunction;
+
+ private transient TypeInformation<Tuple2<VertexKey, Message>> resultType;
+
+
+ private MessagingUdfWithEdgeValues(MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> messagingFunction,
+ TypeInformation<Tuple2<VertexKey, Message>> resultType)
+ {
+ this.messagingFunction = messagingFunction;
+ this.resultType = resultType;
+ }
+
+ @Override
+ public void coGroup(Iterable<Tuple3<VertexKey, VertexKey, EdgeValue>> edges,
+ Iterable<Tuple2<VertexKey, VertexValue>> state, Collector<Tuple2<VertexKey, Message>> out)
+ throws Exception
+ {
+ final Iterator<Tuple2<VertexKey, VertexValue>> stateIter = state.iterator();
+
+ if (stateIter.hasNext()) {
+ Tuple2<VertexKey, VertexValue> newVertexState = stateIter.next();
+ messagingFunction.set((Iterator<?>) edges.iterator(), out);
+ messagingFunction.sendMessages(newVertexState.f0, newVertexState.f1);
+ }
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ if (getIterationRuntimeContext().getSuperstepNumber() == 1) {
+ this.messagingFunction.init(getIterationRuntimeContext(), true);
+ }
+
+ this.messagingFunction.preSuperstep();
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.messagingFunction.postSuperstep();
+ }
+
+ @Override
+ public TypeInformation<Tuple2<VertexKey, Message>> getProducedType() {
+ return this.resultType;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexUpdateFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexUpdateFunction.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexUpdateFunction.java
new file mode 100644
index 0000000..a5548b7
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexUpdateFunction.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java;
+
+import java.io.Serializable;
+import java.util.Collection;
+
+import org.apache.flink.api.common.aggregators.Aggregator;
+import org.apache.flink.api.common.functions.IterationRuntimeContext;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.types.Value;
+import org.apache.flink.util.Collector;
+
+/**
+ * This class must be extended by functions that compute the state of the vertex depending on the old state and the
+ * incoming messages. The central method is {@link #updateVertex(Comparable, Object, MessageIterator)}, which is
+ * invoked once per vertex per superstep.
+ *
+ * <VertexKey> The vertex key type.
+ * <VertexValue> The vertex value type.
+ * <Message> The message type.
+ */
+public abstract class VertexUpdateFunction<VertexKey extends Comparable<VertexKey>, VertexValue, Message> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ // --------------------------------------------------------------------------------------------
+ // Public API Methods
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * This method is invoked once per vertex per superstep. It receives the current state of the vertex, as well as
+ * the incoming messages. It may set a new vertex state via {@link #setNewVertexValue(Object)}. If the vertex
+ * state is changed, it will trigger the sending of messages via the {@link MessagingFunction}.
+ *
+ * @param vertexKey The key (identifier) of the vertex.
+ * @param vertexValue The value (state) of the vertex.
+ * @param inMessages The incoming messages to this vertex.
+ *
+ * @throws Exception The computation may throw exceptions, which causes the superstep to fail.
+ */
+ public abstract void updateVertex(VertexKey vertexKey, VertexValue vertexValue, MessageIterator<Message> inMessages) throws Exception;
+
+ /**
+ * This method is executed one per superstep before the vertex update function is invoked for each vertex.
+ *
+ * @throws Exception Exceptions in the pre-superstep phase cause the superstep to fail.
+ */
+ public void preSuperstep() throws Exception {}
+
+ /**
+ * This method is executed one per superstep after the vertex update function has been invoked for each vertex.
+ *
+ * @throws Exception Exceptions in the post-superstep phase cause the superstep to fail.
+ */
+ public void postSuperstep() throws Exception {}
+
+ /**
+ * Sets the new value of this vertex. Setting a new value triggers the sending of outgoing messages from this vertex.
+ *
+ * @param newValue The new vertex value.
+ */
+ public void setNewVertexValue(VertexValue newValue) {
+ outVal.f1 = newValue;
+ out.collect(outVal);
+ }
+
+ /**
+ * Gets the number of the superstep, starting at <tt>1</tt>.
+ *
+ * @return The number of the current superstep.
+ */
+ public int getSuperstepNumber() {
+ return this.runtimeContext.getSuperstepNumber();
+ }
+
+ /**
+ * Gets the iteration aggregator registered under the given name. The iteration aggregator is combines
+ * all aggregates globally once per superstep and makes them available in the next superstep.
+ *
+ * @param name The name of the aggregator.
+ * @return The aggregator registered under this name, or null, if no aggregator was registered.
+ */
+ public <T extends Aggregator<?>> T getIterationAggregator(String name) {
+ return this.runtimeContext.<T>getIterationAggregator(name);
+ }
+
+ /**
+ * Get the aggregated value that an aggregator computed in the previous iteration.
+ *
+ * @param name The name of the aggregator.
+ * @return The aggregated value of the previous iteration.
+ */
+ public <T extends Value> T getPreviousIterationAggregate(String name) {
+ return this.runtimeContext.<T>getPreviousIterationAggregate(name);
+ }
+
+ /**
+ * Gets the broadcast data set registered under the given name. Broadcast data sets
+ * are available on all parallel instances of a function. They can be registered via
+ * {@link VertexCentricIteration#addBroadcastSetForUpdateFunction(String, org.apache.flink.api.java.DataSet)}.
+ *
+ * @param name The name under which the broadcast set is registered.
+ * @return The broadcast data set.
+ */
+ public <T> Collection<T> getBroadcastSet(String name) {
+ return this.runtimeContext.<T>getBroadcastVariable(name);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // internal methods
+ // --------------------------------------------------------------------------------------------
+
+ private IterationRuntimeContext runtimeContext;
+
+ private Collector<Tuple2<VertexKey, VertexValue>> out;
+
+ private Tuple2<VertexKey, VertexValue> outVal;
+
+
+ void init(IterationRuntimeContext context) {
+ this.runtimeContext = context;
+ }
+
+ void setOutput(Tuple2<VertexKey, VertexValue> val, Collector<Tuple2<VertexKey, VertexValue>> out) {
+ this.out = out;
+ this.outVal = val;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
new file mode 100644
index 0000000..7574dab
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java.examples;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.spargel.java.MessageIterator;
+import org.apache.flink.spargel.java.MessagingFunction;
+import org.apache.flink.spargel.java.VertexCentricIteration;
+import org.apache.flink.spargel.java.VertexUpdateFunction;
+import org.apache.flink.types.NullValue;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+
+@SuppressWarnings({"serial", "unchecked"})
+public class SpargelConnectedComponents {
+
+ public static void main(String[] args) throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> vertexIds = env.generateSequence(0, 10);
+ DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(0L, 2L), new Tuple2<Long, Long>(2L, 4L), new Tuple2<Long, Long>(4L, 8L),
+ new Tuple2<Long, Long>(1L, 5L), new Tuple2<Long, Long>(3L, 7L), new Tuple2<Long, Long>(3L, 9L));
+
+ DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
+
+ DataSet<Tuple2<Long, Long>> result = initialVertices.runOperation(VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100));
+
+ result.print();
+ env.execute("Spargel Connected Components");
+ }
+
+ public static final class CCUpdater extends VertexUpdateFunction<Long, Long, Long> {
+ @Override
+ public void updateVertex(Long vertexKey, Long vertexValue, MessageIterator<Long> inMessages) {
+ long min = Long.MAX_VALUE;
+ for (long msg : inMessages) {
+ min = Math.min(min, msg);
+ }
+ if (min < vertexValue) {
+ setNewVertexValue(min);
+ }
+ }
+ }
+
+ public static final class CCMessager extends MessagingFunction<Long, Long, Long, NullValue> {
+ @Override
+ public void sendMessages(Long vertexId, Long componentId) {
+ sendMessageToAllNeighbors(componentId);
+ }
+ }
+
+ /**
+ * A map function that takes a Long value and creates a 2-tuple out of it:
+ * <pre>(Long value) -> (value, value)</pre>
+ */
+ public static final class IdAssigner implements MapFunction<Long, Tuple2<Long, Long>> {
+ @Override
+ public Tuple2<Long, Long> map(Long value) {
+ return new Tuple2<Long, Long>(value, value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
new file mode 100644
index 0000000..fccb195
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java.examples;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.spargel.java.MessageIterator;
+import org.apache.flink.spargel.java.MessagingFunction;
+import org.apache.flink.spargel.java.OutgoingEdge;
+import org.apache.flink.spargel.java.VertexCentricIteration;
+import org.apache.flink.spargel.java.VertexUpdateFunction;
+import org.apache.flink.util.Collector;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+
+/**
+ * An implementation of the basic PageRank algorithm in the vertex-centric API (spargel).
+ * In this implementation, the edges carry a weight (the transition probability).
+ */
+@SuppressWarnings("serial")
+public class SpargelPageRank {
+
+ private static final double BETA = 0.85;
+
+
+ public static void main(String[] args) throws Exception {
+ final int numVertices = 100;
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ // enumerate some sample edges and assign an initial uniform probability (rank)
+ DataSet<Tuple2<Long, Double>> intialRanks = env.generateSequence(1, numVertices)
+ .map(new MapFunction<Long, Tuple2<Long, Double>>() {
+ public Tuple2<Long, Double> map(Long value) {
+ return new Tuple2<Long, Double>(value, 1.0/numVertices);
+ }
+ });
+
+ // generate some random edges. the transition probability on each edge is 1/num-out-edges of the source vertex
+ DataSet<Tuple3<Long, Long, Double>> edgesWithProbability = env.generateSequence(1, numVertices)
+ .flatMap(new FlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
+ public void flatMap(Long value, Collector<Tuple3<Long, Long, Double>> out) {
+ int numOutEdges = (int) (Math.random() * (numVertices / 2));
+ for (int i = 0; i < numOutEdges; i++) {
+ long target = (long) (Math.random() * numVertices) + 1;
+ out.collect(new Tuple3<Long, Long, Double>(value, target, 1.0/numOutEdges));
+ }
+ }
+ });
+
+ DataSet<Tuple2<Long, Double>> result = intialRanks.runOperation(
+ VertexCentricIteration.withValuedEdges(edgesWithProbability,
+ new VertexRankUpdater(numVertices, BETA), new RankMessenger(), 20));
+
+ result.print();
+ env.execute("Spargel PageRank");
+ }
+
+ /**
+ * Function that updates the rank of a vertex by summing up the partial ranks from all incoming messages
+ * and then applying the dampening formula.
+ */
+ public static final class VertexRankUpdater extends VertexUpdateFunction<Long, Double, Double> {
+
+ private final long numVertices;
+ private final double beta;
+
+ public VertexRankUpdater(long numVertices, double beta) {
+ this.numVertices = numVertices;
+ this.beta = beta;
+ }
+
+ @Override
+ public void updateVertex(Long vertexKey, Double vertexValue, MessageIterator<Double> inMessages) {
+ double rankSum = 0.0;
+ for (double msg : inMessages) {
+ rankSum += msg;
+ }
+
+ // apply the dampening factor / random jump
+ double newRank = (beta * rankSum) + (1-BETA)/numVertices;
+ setNewVertexValue(newRank);
+ }
+ }
+
+ /**
+ * Distributes the rank of a vertex among all target vertices according to the transition probability,
+ * which is associated with an edge as the edge value.
+ */
+ public static final class RankMessenger extends MessagingFunction<Long, Double, Double, Double> {
+
+ @Override
+ public void sendMessages(Long vertexId, Double newRank) {
+ for (OutgoingEdge<Long, Double> edge : getOutgoingEdges()) {
+ sendMessageTo(edge.target(), newRank * edge.edgeValue());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
new file mode 100644
index 0000000..ae4ee95
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java.examples;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.spargel.java.MessageIterator;
+import org.apache.flink.spargel.java.MessagingFunction;
+import org.apache.flink.spargel.java.OutgoingEdge;
+import org.apache.flink.spargel.java.VertexCentricIteration;
+import org.apache.flink.spargel.java.VertexUpdateFunction;
+import org.apache.flink.util.Collector;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+
+/**
+ * An implementation of the basic PageRank algorithm in the vertex-centric API (spargel).
+ * In this implementation, the edges carry a weight (the transition probability).
+ */
+@SuppressWarnings("serial")
+public class SpargelPageRankCountingVertices {
+
+ private static final double BETA = 0.85;
+
+
+ public static void main(String[] args) throws Exception {
+ final int NUM_VERTICES = 100;
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ // a list of vertices
+ DataSet<Long> vertices = env.generateSequence(1, NUM_VERTICES);
+
+ // generate some random edges. the transition probability on each edge is 1/num-out-edges of the source vertex
+ DataSet<Tuple3<Long, Long, Double>> edgesWithProbability = env.generateSequence(1, NUM_VERTICES)
+ .flatMap(new FlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
+ public void flatMap(Long value, Collector<Tuple3<Long, Long, Double>> out) {
+ int numOutEdges = (int) (Math.random() * (NUM_VERTICES / 2));
+ for (int i = 0; i < numOutEdges; i++) {
+ long target = (long) (Math.random() * NUM_VERTICES) + 1;
+ out.collect(new Tuple3<Long, Long, Double>(value, target, 1.0/numOutEdges));
+ }
+ }
+ });
+
+ // ---------- start of the algorithm ---------------
+
+ // count the number of vertices
+ DataSet<Long> count = vertices
+ .map(new MapFunction<Long, Long>() {
+ public Long map(Long value) {
+ return 1L;
+ }
+ })
+ .reduce(new ReduceFunction<Long>() {
+ public Long reduce(Long value1, Long value2) {
+ return value1 + value2;
+ }
+ });
+
+ // enumerate some sample edges and assign an initial uniform probability (rank)
+ DataSet<Tuple2<Long, Double>> intialRanks = vertices
+ .map(new RichMapFunction<Long, Tuple2<Long, Double>>() {
+
+ private long numVertices;
+
+ @Override
+ public void open(Configuration parameters) {
+ numVertices = getRuntimeContext().<Long>getBroadcastVariable("count").iterator().next();
+ }
+
+ public Tuple2<Long, Double> map(Long value) {
+ return new Tuple2<Long, Double>(value, 1.0/numVertices);
+ }
+ }).withBroadcastSet(count, "count");
+
+
+ VertexCentricIteration<Long, Double, Double, Double> iteration = VertexCentricIteration.withValuedEdges(edgesWithProbability,
+ new VertexRankUpdater(BETA), new RankMessenger(), 20);
+ iteration.addBroadcastSetForUpdateFunction("count", count);
+
+
+ DataSet<Tuple2<Long, Double>> result = intialRanks.runOperation(iteration);
+
+ result.print();
+ env.execute("Spargel PageRank");
+ }
+
+ /**
+ * Function that updates the rank of a vertex by summing up the partial ranks from all incoming messages
+ * and then applying the dampening formula.
+ */
+ public static final class VertexRankUpdater extends VertexUpdateFunction<Long, Double, Double> {
+
+ private final double beta;
+ private long numVertices;
+
+ public VertexRankUpdater(double beta) {
+ this.beta = beta;
+ }
+
+ @Override
+ public void preSuperstep() {
+ numVertices = this.<Long>getBroadcastSet("count").iterator().next();
+ }
+
+ @Override
+ public void updateVertex(Long vertexKey, Double vertexValue, MessageIterator<Double> inMessages) {
+ double rankSum = 0.0;
+ for (double msg : inMessages) {
+ rankSum += msg;
+ }
+
+ // apply the dampening factor / random jump
+ double newRank = (beta * rankSum) + (1-BETA)/numVertices;
+ setNewVertexValue(newRank);
+ }
+ }
+
+ /**
+ * Distributes the rank of a vertex among all target vertices according to the transition probability,
+ * which is associated with an edge as the edge value.
+ */
+ public static final class RankMessenger extends MessagingFunction<Long, Double, Double, Double> {
+
+ @Override
+ public void sendMessages(Long vertexId, Double newRank) {
+ for (OutgoingEdge<Long, Double> edge : getOutgoingEdges()) {
+ sendMessageTo(edge.target(), newRank * edge.edgeValue());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/Edge.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/Edge.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/Edge.java
new file mode 100644
index 0000000..0c39688
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/Edge.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java.record;
+
+
+import org.apache.flink.types.Key;
+import org.apache.flink.types.Value;
+
+
+public final class Edge<VertexKey extends Key<VertexKey>, EdgeValue extends Value> {
+
+ private VertexKey target;
+ private EdgeValue edgeValue;
+
+ void set(VertexKey target, EdgeValue edgeValue) {
+ this.target = target;
+ this.edgeValue = edgeValue;
+ }
+
+ public VertexKey target() {
+ return target;
+ }
+
+ public EdgeValue edgeValue() {
+ return edgeValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessageIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessageIterator.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessageIterator.java
new file mode 100644
index 0000000..03022d2
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessageIterator.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java.record;
+
+import java.util.Iterator;
+
+import org.apache.flink.types.Record;
+import org.apache.flink.types.Value;
+
+public final class MessageIterator<Message extends Value> implements Iterator<Message>, Iterable<Message> {
+
+ private final Message instance;
+ private Iterator<Record> source;
+
+ public MessageIterator(Message instance) {
+ this.instance = instance;
+ }
+
+ public final void setSource(Iterator<Record> source) {
+ this.source = source;
+ }
+
+ @Override
+ public final boolean hasNext() {
+ return this.source.hasNext();
+ }
+
+ @Override
+ public final Message next() {
+ this.source.next().getFieldInto(1, this.instance);
+ return this.instance;
+ }
+
+ @Override
+ public final void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterator<Message> iterator() {
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessagingFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessagingFunction.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessagingFunction.java
new file mode 100644
index 0000000..794dce5
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessagingFunction.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java.record;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+import org.apache.flink.api.common.aggregators.Aggregator;
+import org.apache.flink.api.common.functions.IterationRuntimeContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.types.Key;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.Value;
+import org.apache.flink.util.Collector;
+
+public abstract class MessagingFunction<VertexKey extends Key<VertexKey>, VertexValue extends Value, Message extends Value, EdgeValue extends Value> implements Serializable {
+
+ // --------------------------------------------------------------------------------------------
+ // Public API Methods
+ // --------------------------------------------------------------------------------------------
+
+ public abstract void sendMessages(VertexKey vertexKey, VertexValue vertexValue) throws Exception;
+
+ public void setup(Configuration config) throws Exception {}
+
+ public void preSuperstep() throws Exception {}
+
+ public void postSuperstep() throws Exception {}
+
+
+ public Iterator<Edge<VertexKey, EdgeValue>> getOutgoingEdges() {
+ if (edgesUsed) {
+ throw new IllegalStateException("Can use either 'getOutgoingEdges()' or 'sendMessageToAllTargets()'.");
+ }
+
+ edgesUsed = true;
+ edgeIter.set(edges);
+ return edgeIter;
+ }
+
+ public void sendMessageToAllNeighbors(Message m) {
+ if (edgesUsed) {
+ throw new IllegalStateException("Can use either 'getOutgoingEdges()' or 'sendMessageToAllTargets()'.");
+ }
+
+ edgesUsed = true;
+ while (edges.hasNext()) {
+ Record next = edges.next();
+ VertexKey k = next.getField(1, this.keyClass);
+ outValue.setField(0, k);
+ outValue.setField(1, m);
+ out.collect(outValue);
+ }
+ }
+
+ public void sendMessageTo(VertexKey target, Message m) {
+ outValue.setField(0, target);
+ outValue.setField(1, m);
+ out.collect(outValue);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public int getSuperstep() {
+ return this.runtimeContext.getSuperstepNumber();
+ }
+
+ public <T extends Aggregator<?>> T getIterationAggregator(String name) {
+ return this.runtimeContext.<T>getIterationAggregator(name);
+ }
+
+ public <T extends Value> T getPreviousIterationAggregate(String name) {
+ return this.runtimeContext.<T>getPreviousIterationAggregate(name);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // internal methods and state
+ // --------------------------------------------------------------------------------------------
+
+ private Record outValue;
+
+ private IterationRuntimeContext runtimeContext;
+
+ private Iterator<Record> edges;
+
+ private Collector<Record> out;
+
+ private EdgesIterator<VertexKey, EdgeValue> edgeIter;
+
+ private Class<VertexKey> keyClass;
+
+ private boolean edgesUsed;
+
+
+ @SuppressWarnings("unchecked")
+ void init(IterationRuntimeContext context, VertexKey keyHolder, EdgeValue edgeValueHolder) {
+ this.runtimeContext = context;
+ this.edgeIter = new EdgesIterator<VertexKey, EdgeValue>(keyHolder, edgeValueHolder);
+ this.outValue = new Record();
+ this.keyClass = (Class<VertexKey>) keyHolder.getClass();
+ }
+
+ void set(Iterator<Record> edges, Collector<Record> out) {
+ this.edges = edges;
+ this.out = out;
+ this.edgesUsed = false;
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ private static final class EdgesIterator<VertexKey extends Key<VertexKey>, EdgeValue extends Value> implements Iterator<Edge<VertexKey, EdgeValue>> {
+
+ private Iterator<Record> input;
+ private VertexKey keyHolder;
+ private EdgeValue edgeValueHolder;
+
+ private Edge<VertexKey, EdgeValue> edge = new Edge<VertexKey, EdgeValue>();
+
+ EdgesIterator(VertexKey keyHolder, EdgeValue edgeValueHolder) {
+ this.keyHolder = keyHolder;
+ this.edgeValueHolder = edgeValueHolder;
+ }
+
+ void set(Iterator<Record> input) {
+ this.input = input;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return input.hasNext();
+ }
+
+ @Override
+ public Edge<VertexKey, EdgeValue> next() {
+ Record next = input.next();
+ next.getFieldInto(0, keyHolder);
+ next.getFieldInto(1, edgeValueHolder);
+ edge.set(keyHolder, edgeValueHolder);
+ return edge;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java
new file mode 100644
index 0000000..f647e5d
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java.record;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.flink.api.common.aggregators.AggregatorRegistry;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.java.record.functions.CoGroupFunction;
+import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFieldsFirst;
+import org.apache.flink.api.java.record.operators.CoGroupOperator;
+import org.apache.flink.api.java.record.operators.DeltaIteration;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.types.Key;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.Value;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.ReflectionUtil;
+
+@SuppressWarnings("deprecation")
+public class SpargelIteration {
+
+ private static final String DEFAULT_NAME = "<unnamed vertex-centric iteration>";
+
+ private final DeltaIteration iteration;
+
+ private final Class<? extends Key<?>> vertexKey;
+ private final Class<? extends Value> vertexValue;
+ private final Class<? extends Value> messageType;
+ private final Class<? extends Value> edgeValue;
+
+ private final CoGroupOperator vertexUpdater;
+ private final CoGroupOperator messager;
+
+
+ // ----------------------------------------------------------------------------------
+
+ public <VertexKey extends Key<VertexKey>, VertexValue extends Value, Message extends Value, EdgeValue extends Value>
+ SpargelIteration(MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf,
+ VertexUpdateFunction<VertexKey, VertexValue, Message> uf)
+ {
+ this(mf, uf, DEFAULT_NAME);
+ }
+
+ public <VertexKey extends Key<VertexKey>, VertexValue extends Value, Message extends Value, EdgeValue extends Value> SpargelIteration(
+ MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf, VertexUpdateFunction<VertexKey, VertexValue, Message> uf,
+ String name)
+ {
+ // get the types
+ this.vertexKey = ReflectionUtil.getTemplateType1(mf.getClass());
+ this.vertexValue = ReflectionUtil.getTemplateType2(mf.getClass());
+ this.messageType = ReflectionUtil.getTemplateType3(mf.getClass());
+ this.edgeValue = ReflectionUtil.getTemplateType4(mf.getClass());
+
+ if (vertexKey == null || vertexValue == null || messageType == null || edgeValue == null) {
+ throw new RuntimeException();
+ }
+
+ // instantiate the data flow
+ this.iteration = new DeltaIteration(0, name);
+
+ this.messager = CoGroupOperator.builder(MessagingDriver.class, vertexKey, 0, 0)
+ .input2(iteration.getWorkset())
+ .name("Message Sender")
+ .build();
+ this.vertexUpdater = CoGroupOperator.builder(VertexUpdateDriver.class, vertexKey, 0, 0)
+ .input1(messager)
+ .input2(iteration.getSolutionSet())
+ .name("Vertex Updater")
+ .build();
+
+ iteration.setNextWorkset(vertexUpdater);
+ iteration.setSolutionSetDelta(vertexUpdater);
+
+ // parameterize the data flow
+ try {
+ Configuration vertexUdfParams = vertexUpdater.getParameters();
+ InstantiationUtil.writeObjectToConfig(uf, vertexUdfParams, VertexUpdateDriver.UDF_PARAM);
+ vertexUdfParams.setClass(VertexUpdateDriver.KEY_PARAM, vertexKey);
+ vertexUdfParams.setClass(VertexUpdateDriver.VALUE_PARAM, vertexValue);
+ vertexUdfParams.setClass(VertexUpdateDriver.MESSAGE_PARAM, messageType);
+
+ Configuration messageUdfParams = messager.getParameters();
+ InstantiationUtil.writeObjectToConfig(mf, messageUdfParams, MessagingDriver.UDF_PARAM);
+ messageUdfParams.setClass(MessagingDriver.KEY_PARAM, vertexKey);
+ messageUdfParams.setClass(MessagingDriver.VALUE_PARAM, vertexValue);
+ messageUdfParams.setClass(MessagingDriver.MESSAGE_PARAM, messageType);
+ messageUdfParams.setClass(MessagingDriver.EDGE_PARAM, edgeValue);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Could not serialize the UDFs for distribution" +
+ (e.getMessage() == null ? '.' : ": " + e.getMessage()), e);
+ }
+ }
+
+ // ----------------------------------------------------------------------------------
+ // inputs and outputs
+ // ----------------------------------------------------------------------------------
+
+ public void setVertexInput(Operator<Record> c) {
+ this.iteration.setInitialSolutionSet(c);
+ this.iteration.setInitialWorkset(c);
+ }
+
+ public void setEdgesInput(Operator<Record> c) {
+ this.messager.setFirstInput(c);
+ }
+
+ public Operator<?> getOutput() {
+ return this.iteration;
+ }
+
+ public void setDegreeOfParallelism(int dop) {
+ this.iteration.setDegreeOfParallelism(dop);
+ }
+
+ public void setNumberOfIterations(int iterations) {
+ this.iteration.setMaximumNumberOfIterations(iterations);
+ }
+
+ public AggregatorRegistry getAggregators() {
+ return this.iteration.getAggregators();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Wrapping UDFs
+ // --------------------------------------------------------------------------------------------
+
+ @ConstantFieldsFirst(0)
+ public static final class VertexUpdateDriver<K extends Key<K>, V extends Value, M extends Value> extends CoGroupFunction {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final String UDF_PARAM = "spargel.udf";
+ private static final String KEY_PARAM = "spargel.key-type";
+ private static final String VALUE_PARAM = "spargel.value-type";
+ private static final String MESSAGE_PARAM = "spargel.message-type";
+
+ private VertexUpdateFunction<K, V, M> vertexUpdateFunction;
+
+ private K vertexKey;
+ private V vertexValue;
+ private MessageIterator<M> messageIter;
+
+ @Override
+ public void coGroup(Iterator<Record> messages, Iterator<Record> vertex, Collector<Record> out) throws Exception {
+
+ if (vertex.hasNext()) {
+ Record first = vertex.next();
+ first.getFieldInto(0, vertexKey);
+ first.getFieldInto(1, vertexValue);
+ messageIter.setSource(messages);
+ vertexUpdateFunction.setOutput(first, out);
+ vertexUpdateFunction.updateVertex(vertexKey, vertexValue, messageIter);
+ } else {
+ if (messages.hasNext()) {
+ String message = "Target vertex does not exist!.";
+ try {
+ Record next = messages.next();
+ next.getFieldInto(0, vertexKey);
+ message = "Target vertex '" + vertexKey + "' does not exist!.";
+ } catch (Throwable t) {}
+ throw new Exception(message);
+ } else {
+ throw new Exception();
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ // instantiate only the first time
+ if (vertexUpdateFunction == null) {
+ ClassLoader cl = getRuntimeContext().getUserCodeClassLoader();
+
+ Class<K> vertexKeyClass = parameters.getClass(KEY_PARAM, null, cl);
+ Class<V> vertexValueClass = parameters.getClass(VALUE_PARAM, null, cl);
+ Class<M> messageClass = parameters.getClass(MESSAGE_PARAM, null, cl);
+
+ vertexKey = InstantiationUtil.instantiate(vertexKeyClass, Key.class);
+ vertexValue = InstantiationUtil.instantiate(vertexValueClass, Value.class);
+ messageIter = new MessageIterator<M>(InstantiationUtil.instantiate(messageClass, Value.class));
+
+ ClassLoader ucl = getRuntimeContext().getUserCodeClassLoader();
+
+ try {
+ this.vertexUpdateFunction = (VertexUpdateFunction<K, V, M>) InstantiationUtil.readObjectFromConfig(parameters, UDF_PARAM, ucl);
+ } catch (Exception e) {
+ String message = e.getMessage() == null ? "." : ": " + e.getMessage();
+ throw new Exception("Could not instantiate VertexUpdateFunction" + message, e);
+ }
+
+ this.vertexUpdateFunction.init(getIterationRuntimeContext());
+ this.vertexUpdateFunction.setup(parameters);
+ }
+ this.vertexUpdateFunction.preSuperstep();
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.vertexUpdateFunction.postSuperstep();
+ }
+ }
+
+ public static final class MessagingDriver<K extends Key<K>, V extends Value, M extends Value, E extends Value> extends CoGroupFunction {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final String UDF_PARAM = "spargel.udf";
+ private static final String KEY_PARAM = "spargel.key-type";
+ private static final String VALUE_PARAM = "spargel.value-type";
+ private static final String MESSAGE_PARAM = "spargel.message-type";
+ private static final String EDGE_PARAM = "spargel.edge-value";
+
+
+ private MessagingFunction<K, V, M, E> messagingFunction;
+
+ private K vertexKey;
+ private V vertexValue;
+
+ @Override
+ public void coGroup(Iterator<Record> edges, Iterator<Record> state, Collector<Record> out) throws Exception {
+ if (state.hasNext()) {
+ Record first = state.next();
+ first.getFieldInto(0, vertexKey);
+ first.getFieldInto(1, vertexValue);
+ messagingFunction.set(edges, out);
+ messagingFunction.sendMessages(vertexKey, vertexValue);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ // instantiate only the first time
+ if (messagingFunction == null) {
+ ClassLoader cl = getRuntimeContext().getUserCodeClassLoader();
+
+ Class<K> vertexKeyClass = parameters.getClass(KEY_PARAM, null, cl);
+ Class<V> vertexValueClass = parameters.getClass(VALUE_PARAM, null, cl);
+// Class<M> messageClass = parameters.getClass(MESSAGE_PARAM, null, Value.class);
+ Class<E> edgeClass = parameters.getClass(EDGE_PARAM, null, cl);
+
+ vertexKey = InstantiationUtil.instantiate(vertexKeyClass, Key.class);
+ vertexValue = InstantiationUtil.instantiate(vertexValueClass, Value.class);
+
+ K edgeKeyHolder = InstantiationUtil.instantiate(vertexKeyClass, Key.class);
+ E edgeValueHolder = InstantiationUtil.instantiate(edgeClass, Value.class);
+
+ ClassLoader ucl = getRuntimeContext().getUserCodeClassLoader();
+
+ try {
+ this.messagingFunction = (MessagingFunction<K, V, M, E>) InstantiationUtil.readObjectFromConfig(parameters, UDF_PARAM, ucl);
+ } catch (Exception e) {
+ String message = e.getMessage() == null ? "." : ": " + e.getMessage();
+ throw new Exception("Could not instantiate MessagingFunction" + message, e);
+ }
+
+ this.messagingFunction.init(getIterationRuntimeContext(), edgeKeyHolder, edgeValueHolder);
+ this.messagingFunction.setup(parameters);
+ }
+ this.messagingFunction.preSuperstep();
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.messagingFunction.postSuperstep();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/VertexUpdateFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/VertexUpdateFunction.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/VertexUpdateFunction.java
new file mode 100644
index 0000000..299b898
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/VertexUpdateFunction.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java.record;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.aggregators.Aggregator;
+import org.apache.flink.api.common.functions.IterationRuntimeContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.types.Key;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.Value;
+import org.apache.flink.util.Collector;
+
+/**
+ *
+ * <VertexKey> The vertex key type.
+ * <VertexValue> The vertex value type.
+ * <Message> The message type.
+ */
+public abstract class VertexUpdateFunction<VertexKey extends Key<VertexKey>, VertexValue extends Value, Message extends Value> implements Serializable {
+
+ // --------------------------------------------------------------------------------------------
+ // Public API Methods
+ // --------------------------------------------------------------------------------------------
+
+ public abstract void updateVertex(VertexKey vertexKey, VertexValue vertexValue, MessageIterator<Message> inMessages) throws Exception;
+
+ public void setup(Configuration config) throws Exception {}
+
+ public void preSuperstep() throws Exception {}
+
+ public void postSuperstep() throws Exception {}
+
+ public void setNewVertexValue(VertexValue newValue) {
+ outVal.setField(1, newValue);
+ out.collect(outVal);
+ }
+
+ public int getSuperstep() {
+ return this.runtimeContext.getSuperstepNumber();
+ }
+
+ public <T extends Aggregator<?>> T getIterationAggregator(String name) {
+ return this.runtimeContext.<T>getIterationAggregator(name);
+ }
+
+ public <T extends Value> T getPreviousIterationAggregate(String name) {
+ return this.runtimeContext.<T>getPreviousIterationAggregate(name);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // internal methods
+ // --------------------------------------------------------------------------------------------
+
+ private IterationRuntimeContext runtimeContext;
+
+ private Collector<Record> out;
+
+ private Record outVal;
+
+
+ void init(IterationRuntimeContext context) {
+ this.runtimeContext = context;
+ }
+
+ void setOutput(Record val, Collector<Record> out) {
+ this.out = out;
+ this.outVal = val;
+ }
+
+ // serializability
+ private static final long serialVersionUID = 1L;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java b/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java
new file mode 100644
index 0000000..4b6afd3
--- /dev/null
+++ b/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.spargel.java;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Test;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.compiler.plan.DualInputPlanNode;
+import org.apache.flink.compiler.plan.OptimizedPlan;
+import org.apache.flink.compiler.plan.PlanNode;
+import org.apache.flink.compiler.plan.SinkPlanNode;
+import org.apache.flink.compiler.plan.WorksetIterationPlanNode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.apache.flink.spargel.java.VertexCentricIteration;
+import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.CCMessager;
+import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.CCUpdater;
+import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.IdAssigner;
+import org.apache.flink.test.compiler.util.CompilerTestBase;
+
+
+public class SpargelCompilerTest extends CompilerTestBase {
+
+ @Test
+ public void testSpargelCompiler() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+ // compose test program
+ {
+ DataSet<Long> vertexIds = env.generateSequence(1, 2);
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
+
+ DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
+ DataSet<Tuple2<Long, Long>> result = initialVertices.runOperation(VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100));
+
+ result.print();
+ }
+
+ Plan p = env.createProgramPlan("Spargel Connected Components");
+ OptimizedPlan op = compileNoStats(p);
+
+ // check the sink
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(DEFAULT_PARALLELISM, sink.getDegreeOfParallelism());
+
+ // check the iteration
+ WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
+ assertEquals(DEFAULT_PARALLELISM, iteration.getDegreeOfParallelism());
+
+ // check the solution set join and the delta
+ PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
+ assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
+
+ DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
+ assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
+ assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
+ assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
+
+ // check the workset set join
+ DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
+ assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
+ assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
+ assertTrue(edgeJoin.getInput1().getTempMode().isCached());
+
+ assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
+
+ // check that the initial partitioning is pushed out of the loop
+ assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
+ assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
+ assertEquals(new FieldList(0), iteration.getInput2().getShipStrategyKeys());
+
+ // check that the initial workset sort is outside the loop
+ assertEquals(LocalStrategy.SORT, iteration.getInput2().getLocalStrategy());
+ assertEquals(new FieldList(0), iteration.getInput2().getLocalStrategyKeys());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testSpargelCompilerWithBroadcastVariable() {
+ try {
+ final String BC_VAR_NAME = "borat variable";
+
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+ // compose test program
+ {
+ DataSet<Long> bcVar = env.fromElements(1L);
+
+ DataSet<Long> vertexIds = env.generateSequence(1, 2);
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
+
+ DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
+
+ VertexCentricIteration<Long, Long, Long, ?> vcIter = VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100);
+ vcIter.addBroadcastSetForMessagingFunction(BC_VAR_NAME, bcVar);
+ vcIter.addBroadcastSetForUpdateFunction(BC_VAR_NAME, bcVar);
+
+ DataSet<Tuple2<Long, Long>> result = initialVertices.runOperation(vcIter);
+
+ result.print();
+ }
+
+ Plan p = env.createProgramPlan("Spargel Connected Components");
+ OptimizedPlan op = compileNoStats(p);
+
+ // check the sink
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(DEFAULT_PARALLELISM, sink.getDegreeOfParallelism());
+
+ // check the iteration
+ WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
+ assertEquals(DEFAULT_PARALLELISM, iteration.getDegreeOfParallelism());
+
+ // check the solution set join and the delta
+ PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
+ assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
+
+ DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
+ assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
+ assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
+ assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
+
+ // check the workset set join
+ DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
+ assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
+ assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
+ assertTrue(edgeJoin.getInput1().getTempMode().isCached());
+
+ assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
+
+ // check that the initial partitioning is pushed out of the loop
+ assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
+ assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
+ assertEquals(new FieldList(0), iteration.getInput2().getShipStrategyKeys());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
[25/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java
new file mode 100644
index 0000000..1174786
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.avro.testjar;
+
+// ================================================================================================
+// This file defines the classes for the AvroExternalJarProgramITCase.
+// The program is exported into src/test/resources/AvroTestProgram.jar.
+//
+// THIS FILE MUST STAY FULLY COMMENTED SUCH THAT THE HERE DEFINED CLASSES ARE NOT COMPILED
+// AND ADDED TO THE test-classes DIRECTORY. OTHERWISE, THE EXTERNAL CLASS LOADING WILL
+// NOT BE COVERED BY THIS TEST.
+// ================================================================================================
+
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.reflect.ReflectDatumWriter;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.functions.RichReduceFunction;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.AvroInputFormat;
+import org.apache.flink.core.fs.Path;
+
+public class AvroExternalJarProgram {
+
+ public static final class Color {
+
+ private String name;
+ private double saturation;
+
+ public Color() {
+ name = "";
+ saturation = 1.0;
+ }
+
+ public Color(String name, double saturation) {
+ this.name = name;
+ this.saturation = saturation;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public double getSaturation() {
+ return saturation;
+ }
+
+ public void setSaturation(double saturation) {
+ this.saturation = saturation;
+ }
+
+ @Override
+ public String toString() {
+ return name + '(' + saturation + ')';
+ }
+ }
+
+ public static final class MyUser {
+
+ private String name;
+ private List<Color> colors;
+
+ public MyUser() {
+ name = "unknown";
+ colors = new ArrayList<Color>();
+ }
+
+ public MyUser(String name, List<Color> colors) {
+ this.name = name;
+ this.colors = colors;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public List<Color> getColors() {
+ return colors;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void setColors(List<Color> colors) {
+ this.colors = colors;
+ }
+
+ @Override
+ public String toString() {
+ return name + " : " + colors;
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ // --------------------------------------------------------------------------------------------
+
+ public static final class NameExtractor extends RichMapFunction<MyUser, Tuple2<String, MyUser>> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<String, MyUser> map(MyUser u) {
+ String namePrefix = u.getName().substring(0, 1);
+ return new Tuple2<String, MyUser>(namePrefix, u);
+ }
+ }
+
+ public static final class NameGrouper extends RichReduceFunction<Tuple2<String, MyUser>> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<String, MyUser> reduce(Tuple2<String, MyUser> val1, Tuple2<String, MyUser> val2) {
+ return val1;
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Test Data
+ // --------------------------------------------------------------------------------------------
+
+ public static final class Generator {
+
+ private final Random rnd = new Random(2389756789345689276L);
+
+ public MyUser nextUser() {
+ return randomUser();
+ }
+
+ private MyUser randomUser() {
+
+ int numColors = rnd.nextInt(5);
+ ArrayList<Color> colors = new ArrayList<Color>(numColors);
+ for (int i = 0; i < numColors; i++) {
+ colors.add(new Color(randomString(), rnd.nextDouble()));
+ }
+
+ return new MyUser(randomString(), colors);
+ }
+
+ private String randomString() {
+ char[] c = new char[this.rnd.nextInt(20) + 5];
+
+ for (int i = 0; i < c.length; i++) {
+ c[i] = (char) (this.rnd.nextInt(150) + 40);
+ }
+
+ return new String(c);
+ }
+ }
+
+ public static void writeTestData(File testFile, int numRecords) throws IOException {
+
+ DatumWriter<MyUser> userDatumWriter = new ReflectDatumWriter<MyUser>(MyUser.class);
+ DataFileWriter<MyUser> dataFileWriter = new DataFileWriter<MyUser>(userDatumWriter);
+
+ dataFileWriter.create(ReflectData.get().getSchema(MyUser.class), testFile);
+
+
+ Generator generator = new Generator();
+
+ for (int i = 0; i < numRecords; i++) {
+ MyUser user = generator.nextUser();
+ dataFileWriter.append(user);
+ }
+
+ dataFileWriter.close();
+ }
+
+// public static void main(String[] args) throws Exception {
+// String testDataFile = new File("src/test/resources/testdata.avro").getAbsolutePath();
+// writeTestData(new File(testDataFile), 50);
+// }
+
+ public static void main(String[] args) throws Exception {
+ String inputPath = args[0];
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<MyUser> input = env.createInput(new AvroInputFormat<MyUser>(new Path(inputPath), MyUser.class));
+
+ DataSet<Tuple2<String, MyUser>> result = input.map(new NameExtractor()).groupBy(0).reduce(new NameGrouper());
+
+ result.output(new DiscardingOutputFormat<Tuple2<String,MyUser>>());
+ env.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
new file mode 100644
index 0000000..d8d8b46
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.io.avro;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.avro.util.Utf8;
+import org.apache.flink.api.io.avro.generated.Colors;
+import org.apache.flink.api.io.avro.generated.User;
+import org.apache.flink.api.java.io.AvroInputFormat;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.FileInputSplit;
+import org.apache.flink.core.fs.Path;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test the avro input format.
+ * (The testcase is mostly the getting started tutorial of avro)
+ * http://avro.apache.org/docs/current/gettingstartedjava.html
+ */
+public class AvroRecordInputFormatTest {
+
+ private File testFile;
+
+ final static String TEST_NAME = "Alyssa";
+
+ final static String TEST_ARRAY_STRING_1 = "ELEMENT 1";
+ final static String TEST_ARRAY_STRING_2 = "ELEMENT 2";
+
+ final static boolean TEST_ARRAY_BOOLEAN_1 = true;
+ final static boolean TEST_ARRAY_BOOLEAN_2 = false;
+
+ final static Colors TEST_ENUM_COLOR = Colors.GREEN;
+
+ final static String TEST_MAP_KEY1 = "KEY 1";
+ final static long TEST_MAP_VALUE1 = 8546456L;
+ final static String TEST_MAP_KEY2 = "KEY 2";
+ final static long TEST_MAP_VALUE2 = 17554L;
+
+ @Before
+ public void createFiles() throws IOException {
+ testFile = File.createTempFile("AvroInputFormatTest", null);
+
+ ArrayList<CharSequence> stringArray = new ArrayList<CharSequence>();
+ stringArray.add(TEST_ARRAY_STRING_1);
+ stringArray.add(TEST_ARRAY_STRING_2);
+
+ ArrayList<Boolean> booleanArray = new ArrayList<Boolean>();
+ booleanArray.add(TEST_ARRAY_BOOLEAN_1);
+ booleanArray.add(TEST_ARRAY_BOOLEAN_2);
+
+ HashMap<CharSequence, Long> longMap = new HashMap<CharSequence, Long>();
+ longMap.put(TEST_MAP_KEY1, TEST_MAP_VALUE1);
+ longMap.put(TEST_MAP_KEY2, TEST_MAP_VALUE2);
+
+
+ User user1 = new User();
+ user1.setName(TEST_NAME);
+ user1.setFavoriteNumber(256);
+ user1.setTypeDoubleTest(123.45d);
+ user1.setTypeBoolTest(true);
+ user1.setTypeArrayString(stringArray);
+ user1.setTypeArrayBoolean(booleanArray);
+ user1.setTypeEnum(TEST_ENUM_COLOR);
+ user1.setTypeMap(longMap);
+
+ // Construct via builder
+ User user2 = User.newBuilder()
+ .setName("Charlie")
+ .setFavoriteColor("blue")
+ .setFavoriteNumber(null)
+ .setTypeBoolTest(false)
+ .setTypeDoubleTest(1.337d)
+ .setTypeNullTest(null)
+ .setTypeLongTest(1337L)
+ .setTypeArrayString(new ArrayList<CharSequence>())
+ .setTypeArrayBoolean(new ArrayList<Boolean>())
+ .setTypeNullableArray(null)
+ .setTypeEnum(Colors.RED)
+ .setTypeMap(new HashMap<CharSequence, Long>())
+ .build();
+ DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
+ DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);
+ dataFileWriter.create(user1.getSchema(), testFile);
+ dataFileWriter.append(user1);
+ dataFileWriter.append(user2);
+ dataFileWriter.close();
+ }
+
+ @Test
+ public void testDeserialisation() throws IOException {
+ Configuration parameters = new Configuration();
+
+ AvroInputFormat<User> format = new AvroInputFormat<User>(new Path(testFile.getAbsolutePath()), User.class);
+
+ format.configure(parameters);
+ FileInputSplit[] splits = format.createInputSplits(1);
+ assertEquals(splits.length, 1);
+ format.open(splits[0]);
+
+ User u = format.nextRecord(null);
+ assertNotNull(u);
+
+ String name = u.getName().toString();
+ assertNotNull("empty record", name);
+ assertEquals("name not equal", TEST_NAME, name);
+
+ // check arrays
+ List<CharSequence> sl = u.getTypeArrayString();
+ assertEquals("element 0 not equal", TEST_ARRAY_STRING_1, sl.get(0).toString());
+ assertEquals("element 1 not equal", TEST_ARRAY_STRING_2, sl.get(1).toString());
+
+ List<Boolean> bl = u.getTypeArrayBoolean();
+ assertEquals("element 0 not equal", TEST_ARRAY_BOOLEAN_1, bl.get(0));
+ assertEquals("element 1 not equal", TEST_ARRAY_BOOLEAN_2, bl.get(1));
+
+ // check enums
+ Colors enumValue = u.getTypeEnum();
+ assertEquals("enum not equal", TEST_ENUM_COLOR, enumValue);
+
+ // check maps
+ Map<CharSequence, Long> lm = u.getTypeMap();
+ assertEquals("map value of key 1 not equal", TEST_MAP_VALUE1, lm.get(new Utf8(TEST_MAP_KEY1)).longValue());
+ assertEquals("map value of key 2 not equal", TEST_MAP_VALUE2, lm.get(new Utf8(TEST_MAP_KEY2)).longValue());
+
+ assertFalse("expecting second element", format.reachedEnd());
+ assertNotNull("expecting second element", format.nextRecord(u));
+
+ assertNull(format.nextRecord(u));
+ assertTrue(format.reachedEnd());
+
+ format.close();
+ }
+
+ @After
+ public void deleteFiles() {
+ testFile.delete();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/Colors.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/Colors.java b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/Colors.java
new file mode 100644
index 0000000..58e1f5c
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/Colors.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * Autogenerated by Avro
+ *
+ * DO NOT EDIT DIRECTLY
+ */
+package org.apache.flink.api.io.avro.generated;
+@SuppressWarnings("all")
+@org.apache.avro.specific.AvroGenerated
+public enum Colors {
+ RED, GREEN, BLUE ;
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"enum\",\"name\":\"Colors\",\"namespace\":\"org.apache.flink.api.io.avro.generated\",\"symbols\":[\"RED\",\"GREEN\",\"BLUE\"]}");
+ public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/User.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/User.java b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/User.java
new file mode 100644
index 0000000..505857e
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/User.java
@@ -0,0 +1,755 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * Autogenerated by Avro
+ *
+ * DO NOT EDIT DIRECTLY
+ */
+package org.apache.flink.api.io.avro.generated;
+@SuppressWarnings("all")
+@org.apache.avro.specific.AvroGenerated
+public class User extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"org.apache.flink.api.io.avro.generated\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]},{\"name\":\"type_long_test\",\"type\":[\"long\",\"null\"]},{\"name\":\"type_double_test\",\"type\":[\"double\"]},{\"name\":\"type_null_test\",\"type\":[\"null\"]},{\"name\":\"type_bool_test\",\"type\":[\"boolean\"]},{\"name\":\"type_array_string\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"type_array_boolean\",\"type\":{\"type\":\"array\",\"items\":\"boolean\"}},{\"name\":\"type_nullable_array\",\"type\":[\"null\",{\"type\":\"array\",\"items\":\"string\"}],\"default\":null},{\"name\":\"type_enum\",\"type\":{\"type\":\"enum\",\"name\":\"Colors\",\"symbols\":[\"RED\",\"GREEN\",\"BLUE\"]}},{\"name\":\"type
_map\",\"type\":{\"type\":\"map\",\"values\":\"long\"}}]}");
+ public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
+ @Deprecated public java.lang.CharSequence name;
+ @Deprecated public java.lang.Integer favorite_number;
+ @Deprecated public java.lang.CharSequence favorite_color;
+ @Deprecated public java.lang.Long type_long_test;
+ @Deprecated public java.lang.Object type_double_test;
+ @Deprecated public java.lang.Object type_null_test;
+ @Deprecated public java.lang.Object type_bool_test;
+ @Deprecated public java.util.List<java.lang.CharSequence> type_array_string;
+ @Deprecated public java.util.List<java.lang.Boolean> type_array_boolean;
+ @Deprecated public java.util.List<java.lang.CharSequence> type_nullable_array;
+ @Deprecated public org.apache.flink.api.io.avro.generated.Colors type_enum;
+ @Deprecated public java.util.Map<java.lang.CharSequence,java.lang.Long> type_map;
+
+ /**
+ * Default constructor. Note that this does not initialize fields
+ * to their default values from the schema. If that is desired then
+ * one should use {@link \#newBuilder()}.
+ */
+ public User() {}
+
+ /**
+ * All-args constructor.
+ */
+ public User(java.lang.CharSequence name, java.lang.Integer favorite_number, java.lang.CharSequence favorite_color, java.lang.Long type_long_test, java.lang.Object type_double_test, java.lang.Object type_null_test, java.lang.Object type_bool_test, java.util.List<java.lang.CharSequence> type_array_string, java.util.List<java.lang.Boolean> type_array_boolean, java.util.List<java.lang.CharSequence> type_nullable_array, org.apache.flink.api.io.avro.generated.Colors type_enum, java.util.Map<java.lang.CharSequence,java.lang.Long> type_map) {
+ this.name = name;
+ this.favorite_number = favorite_number;
+ this.favorite_color = favorite_color;
+ this.type_long_test = type_long_test;
+ this.type_double_test = type_double_test;
+ this.type_null_test = type_null_test;
+ this.type_bool_test = type_bool_test;
+ this.type_array_string = type_array_string;
+ this.type_array_boolean = type_array_boolean;
+ this.type_nullable_array = type_nullable_array;
+ this.type_enum = type_enum;
+ this.type_map = type_map;
+ }
+
+ public org.apache.avro.Schema getSchema() { return SCHEMA$; }
+ // Used by DatumWriter. Applications should not call.
+ public java.lang.Object get(int field$) {
+ switch (field$) {
+ case 0: return name;
+ case 1: return favorite_number;
+ case 2: return favorite_color;
+ case 3: return type_long_test;
+ case 4: return type_double_test;
+ case 5: return type_null_test;
+ case 6: return type_bool_test;
+ case 7: return type_array_string;
+ case 8: return type_array_boolean;
+ case 9: return type_nullable_array;
+ case 10: return type_enum;
+ case 11: return type_map;
+ default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+ }
+ }
+ // Used by DatumReader. Applications should not call.
+ @SuppressWarnings(value="unchecked")
+ public void put(int field$, java.lang.Object value$) {
+ switch (field$) {
+ case 0: name = (java.lang.CharSequence)value$; break;
+ case 1: favorite_number = (java.lang.Integer)value$; break;
+ case 2: favorite_color = (java.lang.CharSequence)value$; break;
+ case 3: type_long_test = (java.lang.Long)value$; break;
+ case 4: type_double_test = (java.lang.Object)value$; break;
+ case 5: type_null_test = (java.lang.Object)value$; break;
+ case 6: type_bool_test = (java.lang.Object)value$; break;
+ case 7: type_array_string = (java.util.List<java.lang.CharSequence>)value$; break;
+ case 8: type_array_boolean = (java.util.List<java.lang.Boolean>)value$; break;
+ case 9: type_nullable_array = (java.util.List<java.lang.CharSequence>)value$; break;
+ case 10: type_enum = (org.apache.flink.api.io.avro.generated.Colors)value$; break;
+ case 11: type_map = (java.util.Map<java.lang.CharSequence,java.lang.Long>)value$; break;
+ default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+ }
+ }
+
+ /**
+ * Gets the value of the 'name' field.
+ */
+ public java.lang.CharSequence getName() {
+ return name;
+ }
+
+ /**
+ * Sets the value of the 'name' field.
+ * @param value the value to set.
+ */
+ public void setName(java.lang.CharSequence value) {
+ this.name = value;
+ }
+
+ /**
+ * Gets the value of the 'favorite_number' field.
+ */
+ public java.lang.Integer getFavoriteNumber() {
+ return favorite_number;
+ }
+
+ /**
+ * Sets the value of the 'favorite_number' field.
+ * @param value the value to set.
+ */
+ public void setFavoriteNumber(java.lang.Integer value) {
+ this.favorite_number = value;
+ }
+
+ /**
+ * Gets the value of the 'favorite_color' field.
+ */
+ public java.lang.CharSequence getFavoriteColor() {
+ return favorite_color;
+ }
+
+ /**
+ * Sets the value of the 'favorite_color' field.
+ * @param value the value to set.
+ */
+ public void setFavoriteColor(java.lang.CharSequence value) {
+ this.favorite_color = value;
+ }
+
+ /**
+ * Gets the value of the 'type_long_test' field.
+ */
+ public java.lang.Long getTypeLongTest() {
+ return type_long_test;
+ }
+
+ /**
+ * Sets the value of the 'type_long_test' field.
+ * @param value the value to set.
+ */
+ public void setTypeLongTest(java.lang.Long value) {
+ this.type_long_test = value;
+ }
+
+ /**
+ * Gets the value of the 'type_double_test' field.
+ */
+ public java.lang.Object getTypeDoubleTest() {
+ return type_double_test;
+ }
+
+ /**
+ * Sets the value of the 'type_double_test' field.
+ * @param value the value to set.
+ */
+ public void setTypeDoubleTest(java.lang.Object value) {
+ this.type_double_test = value;
+ }
+
+ /**
+ * Gets the value of the 'type_null_test' field.
+ */
+ public java.lang.Object getTypeNullTest() {
+ return type_null_test;
+ }
+
+ /**
+ * Sets the value of the 'type_null_test' field.
+ * @param value the value to set.
+ */
+ public void setTypeNullTest(java.lang.Object value) {
+ this.type_null_test = value;
+ }
+
+ /**
+ * Gets the value of the 'type_bool_test' field.
+ */
+ public java.lang.Object getTypeBoolTest() {
+ return type_bool_test;
+ }
+
+ /**
+ * Sets the value of the 'type_bool_test' field.
+ * @param value the value to set.
+ */
+ public void setTypeBoolTest(java.lang.Object value) {
+ this.type_bool_test = value;
+ }
+
+ /**
+ * Gets the value of the 'type_array_string' field.
+ */
+ public java.util.List<java.lang.CharSequence> getTypeArrayString() {
+ return type_array_string;
+ }
+
+ /**
+ * Sets the value of the 'type_array_string' field.
+ * @param value the value to set.
+ */
+ public void setTypeArrayString(java.util.List<java.lang.CharSequence> value) {
+ this.type_array_string = value;
+ }
+
+ /**
+ * Gets the value of the 'type_array_boolean' field.
+ */
+ public java.util.List<java.lang.Boolean> getTypeArrayBoolean() {
+ return type_array_boolean;
+ }
+
+ /**
+ * Sets the value of the 'type_array_boolean' field.
+ * @param value the value to set.
+ */
+ public void setTypeArrayBoolean(java.util.List<java.lang.Boolean> value) {
+ this.type_array_boolean = value;
+ }
+
+ /**
+ * Gets the value of the 'type_nullable_array' field.
+ */
+ public java.util.List<java.lang.CharSequence> getTypeNullableArray() {
+ return type_nullable_array;
+ }
+
+ /**
+ * Sets the value of the 'type_nullable_array' field.
+ * @param value the value to set.
+ */
+ public void setTypeNullableArray(java.util.List<java.lang.CharSequence> value) {
+ this.type_nullable_array = value;
+ }
+
+ /**
+ * Gets the value of the 'type_enum' field.
+ */
+ public org.apache.flink.api.io.avro.generated.Colors getTypeEnum() {
+ return type_enum;
+ }
+
+ /**
+ * Sets the value of the 'type_enum' field.
+ * @param value the value to set.
+ */
+ public void setTypeEnum(org.apache.flink.api.io.avro.generated.Colors value) {
+ this.type_enum = value;
+ }
+
+ /**
+ * Gets the value of the 'type_map' field.
+ */
+ public java.util.Map<java.lang.CharSequence,java.lang.Long> getTypeMap() {
+ return type_map;
+ }
+
+ /**
+ * Sets the value of the 'type_map' field.
+ * @param value the value to set.
+ */
+ public void setTypeMap(java.util.Map<java.lang.CharSequence,java.lang.Long> value) {
+ this.type_map = value;
+ }
+
+ /** Creates a new User RecordBuilder */
+ public static org.apache.flink.api.io.avro.generated.User.Builder newBuilder() {
+ return new org.apache.flink.api.io.avro.generated.User.Builder();
+ }
+
+ /** Creates a new User RecordBuilder by copying an existing Builder */
+ public static org.apache.flink.api.io.avro.generated.User.Builder newBuilder(org.apache.flink.api.io.avro.generated.User.Builder other) {
+ return new org.apache.flink.api.io.avro.generated.User.Builder(other);
+ }
+
+ /** Creates a new User RecordBuilder by copying an existing User instance */
+ public static org.apache.flink.api.io.avro.generated.User.Builder newBuilder(org.apache.flink.api.io.avro.generated.User other) {
+ return new org.apache.flink.api.io.avro.generated.User.Builder(other);
+ }
+
+ /**
+ * RecordBuilder for User instances.
+ */
+ public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<User>
+ implements org.apache.avro.data.RecordBuilder<User> {
+
+ private java.lang.CharSequence name;
+ private java.lang.Integer favorite_number;
+ private java.lang.CharSequence favorite_color;
+ private java.lang.Long type_long_test;
+ private java.lang.Object type_double_test;
+ private java.lang.Object type_null_test;
+ private java.lang.Object type_bool_test;
+ private java.util.List<java.lang.CharSequence> type_array_string;
+ private java.util.List<java.lang.Boolean> type_array_boolean;
+ private java.util.List<java.lang.CharSequence> type_nullable_array;
+ private org.apache.flink.api.io.avro.generated.Colors type_enum;
+ private java.util.Map<java.lang.CharSequence,java.lang.Long> type_map;
+
+ /** Creates a new Builder */
+ private Builder() {
+ super(org.apache.flink.api.io.avro.generated.User.SCHEMA$);
+ }
+
+ /** Creates a Builder by copying an existing Builder */
+ private Builder(org.apache.flink.api.io.avro.generated.User.Builder other) {
+ super(other);
+ if (isValidValue(fields()[0], other.name)) {
+ this.name = data().deepCopy(fields()[0].schema(), other.name);
+ fieldSetFlags()[0] = true;
+ }
+ if (isValidValue(fields()[1], other.favorite_number)) {
+ this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);
+ fieldSetFlags()[1] = true;
+ }
+ if (isValidValue(fields()[2], other.favorite_color)) {
+ this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);
+ fieldSetFlags()[2] = true;
+ }
+ if (isValidValue(fields()[3], other.type_long_test)) {
+ this.type_long_test = data().deepCopy(fields()[3].schema(), other.type_long_test);
+ fieldSetFlags()[3] = true;
+ }
+ if (isValidValue(fields()[4], other.type_double_test)) {
+ this.type_double_test = data().deepCopy(fields()[4].schema(), other.type_double_test);
+ fieldSetFlags()[4] = true;
+ }
+ if (isValidValue(fields()[5], other.type_null_test)) {
+ this.type_null_test = data().deepCopy(fields()[5].schema(), other.type_null_test);
+ fieldSetFlags()[5] = true;
+ }
+ if (isValidValue(fields()[6], other.type_bool_test)) {
+ this.type_bool_test = data().deepCopy(fields()[6].schema(), other.type_bool_test);
+ fieldSetFlags()[6] = true;
+ }
+ if (isValidValue(fields()[7], other.type_array_string)) {
+ this.type_array_string = data().deepCopy(fields()[7].schema(), other.type_array_string);
+ fieldSetFlags()[7] = true;
+ }
+ if (isValidValue(fields()[8], other.type_array_boolean)) {
+ this.type_array_boolean = data().deepCopy(fields()[8].schema(), other.type_array_boolean);
+ fieldSetFlags()[8] = true;
+ }
+ if (isValidValue(fields()[9], other.type_nullable_array)) {
+ this.type_nullable_array = data().deepCopy(fields()[9].schema(), other.type_nullable_array);
+ fieldSetFlags()[9] = true;
+ }
+ if (isValidValue(fields()[10], other.type_enum)) {
+ this.type_enum = data().deepCopy(fields()[10].schema(), other.type_enum);
+ fieldSetFlags()[10] = true;
+ }
+ if (isValidValue(fields()[11], other.type_map)) {
+ this.type_map = data().deepCopy(fields()[11].schema(), other.type_map);
+ fieldSetFlags()[11] = true;
+ }
+ }
+
+ /** Creates a Builder by copying an existing User instance */
+ private Builder(org.apache.flink.api.io.avro.generated.User other) {
+ super(org.apache.flink.api.io.avro.generated.User.SCHEMA$);
+ if (isValidValue(fields()[0], other.name)) {
+ this.name = data().deepCopy(fields()[0].schema(), other.name);
+ fieldSetFlags()[0] = true;
+ }
+ if (isValidValue(fields()[1], other.favorite_number)) {
+ this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);
+ fieldSetFlags()[1] = true;
+ }
+ if (isValidValue(fields()[2], other.favorite_color)) {
+ this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);
+ fieldSetFlags()[2] = true;
+ }
+ if (isValidValue(fields()[3], other.type_long_test)) {
+ this.type_long_test = data().deepCopy(fields()[3].schema(), other.type_long_test);
+ fieldSetFlags()[3] = true;
+ }
+ if (isValidValue(fields()[4], other.type_double_test)) {
+ this.type_double_test = data().deepCopy(fields()[4].schema(), other.type_double_test);
+ fieldSetFlags()[4] = true;
+ }
+ if (isValidValue(fields()[5], other.type_null_test)) {
+ this.type_null_test = data().deepCopy(fields()[5].schema(), other.type_null_test);
+ fieldSetFlags()[5] = true;
+ }
+ if (isValidValue(fields()[6], other.type_bool_test)) {
+ this.type_bool_test = data().deepCopy(fields()[6].schema(), other.type_bool_test);
+ fieldSetFlags()[6] = true;
+ }
+ if (isValidValue(fields()[7], other.type_array_string)) {
+ this.type_array_string = data().deepCopy(fields()[7].schema(), other.type_array_string);
+ fieldSetFlags()[7] = true;
+ }
+ if (isValidValue(fields()[8], other.type_array_boolean)) {
+ this.type_array_boolean = data().deepCopy(fields()[8].schema(), other.type_array_boolean);
+ fieldSetFlags()[8] = true;
+ }
+ if (isValidValue(fields()[9], other.type_nullable_array)) {
+ this.type_nullable_array = data().deepCopy(fields()[9].schema(), other.type_nullable_array);
+ fieldSetFlags()[9] = true;
+ }
+ if (isValidValue(fields()[10], other.type_enum)) {
+ this.type_enum = data().deepCopy(fields()[10].schema(), other.type_enum);
+ fieldSetFlags()[10] = true;
+ }
+ if (isValidValue(fields()[11], other.type_map)) {
+ this.type_map = data().deepCopy(fields()[11].schema(), other.type_map);
+ fieldSetFlags()[11] = true;
+ }
+ }
+
+ /** Gets the value of the 'name' field */
+ public java.lang.CharSequence getName() {
+ return name;
+ }
+
+ /** Sets the value of the 'name' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setName(java.lang.CharSequence value) {
+ validate(fields()[0], value);
+ this.name = value;
+ fieldSetFlags()[0] = true;
+ return this;
+ }
+
+ /** Checks whether the 'name' field has been set */
+ public boolean hasName() {
+ return fieldSetFlags()[0];
+ }
+
+ /** Clears the value of the 'name' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearName() {
+ name = null;
+ fieldSetFlags()[0] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'favorite_number' field */
+ public java.lang.Integer getFavoriteNumber() {
+ return favorite_number;
+ }
+
+ /** Sets the value of the 'favorite_number' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setFavoriteNumber(java.lang.Integer value) {
+ validate(fields()[1], value);
+ this.favorite_number = value;
+ fieldSetFlags()[1] = true;
+ return this;
+ }
+
+ /** Checks whether the 'favorite_number' field has been set */
+ public boolean hasFavoriteNumber() {
+ return fieldSetFlags()[1];
+ }
+
+ /** Clears the value of the 'favorite_number' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearFavoriteNumber() {
+ favorite_number = null;
+ fieldSetFlags()[1] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'favorite_color' field */
+ public java.lang.CharSequence getFavoriteColor() {
+ return favorite_color;
+ }
+
+ /** Sets the value of the 'favorite_color' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setFavoriteColor(java.lang.CharSequence value) {
+ validate(fields()[2], value);
+ this.favorite_color = value;
+ fieldSetFlags()[2] = true;
+ return this;
+ }
+
+ /** Checks whether the 'favorite_color' field has been set */
+ public boolean hasFavoriteColor() {
+ return fieldSetFlags()[2];
+ }
+
+ /** Clears the value of the 'favorite_color' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearFavoriteColor() {
+ favorite_color = null;
+ fieldSetFlags()[2] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'type_long_test' field */
+ public java.lang.Long getTypeLongTest() {
+ return type_long_test;
+ }
+
+ /** Sets the value of the 'type_long_test' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setTypeLongTest(java.lang.Long value) {
+ validate(fields()[3], value);
+ this.type_long_test = value;
+ fieldSetFlags()[3] = true;
+ return this;
+ }
+
+ /** Checks whether the 'type_long_test' field has been set */
+ public boolean hasTypeLongTest() {
+ return fieldSetFlags()[3];
+ }
+
+ /** Clears the value of the 'type_long_test' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearTypeLongTest() {
+ type_long_test = null;
+ fieldSetFlags()[3] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'type_double_test' field */
+ public java.lang.Object getTypeDoubleTest() {
+ return type_double_test;
+ }
+
+ /** Sets the value of the 'type_double_test' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setTypeDoubleTest(java.lang.Object value) {
+ validate(fields()[4], value);
+ this.type_double_test = value;
+ fieldSetFlags()[4] = true;
+ return this;
+ }
+
+ /** Checks whether the 'type_double_test' field has been set */
+ public boolean hasTypeDoubleTest() {
+ return fieldSetFlags()[4];
+ }
+
+ /** Clears the value of the 'type_double_test' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearTypeDoubleTest() {
+ type_double_test = null;
+ fieldSetFlags()[4] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'type_null_test' field */
+ public java.lang.Object getTypeNullTest() {
+ return type_null_test;
+ }
+
+ /** Sets the value of the 'type_null_test' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setTypeNullTest(java.lang.Object value) {
+ validate(fields()[5], value);
+ this.type_null_test = value;
+ fieldSetFlags()[5] = true;
+ return this;
+ }
+
+ /** Checks whether the 'type_null_test' field has been set */
+ public boolean hasTypeNullTest() {
+ return fieldSetFlags()[5];
+ }
+
+ /** Clears the value of the 'type_null_test' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearTypeNullTest() {
+ type_null_test = null;
+ fieldSetFlags()[5] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'type_bool_test' field */
+ public java.lang.Object getTypeBoolTest() {
+ return type_bool_test;
+ }
+
+ /** Sets the value of the 'type_bool_test' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setTypeBoolTest(java.lang.Object value) {
+ validate(fields()[6], value);
+ this.type_bool_test = value;
+ fieldSetFlags()[6] = true;
+ return this;
+ }
+
+ /** Checks whether the 'type_bool_test' field has been set */
+ public boolean hasTypeBoolTest() {
+ return fieldSetFlags()[6];
+ }
+
+ /** Clears the value of the 'type_bool_test' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearTypeBoolTest() {
+ type_bool_test = null;
+ fieldSetFlags()[6] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'type_array_string' field */
+ public java.util.List<java.lang.CharSequence> getTypeArrayString() {
+ return type_array_string;
+ }
+
+ /** Sets the value of the 'type_array_string' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setTypeArrayString(java.util.List<java.lang.CharSequence> value) {
+ validate(fields()[7], value);
+ this.type_array_string = value;
+ fieldSetFlags()[7] = true;
+ return this;
+ }
+
+ /** Checks whether the 'type_array_string' field has been set */
+ public boolean hasTypeArrayString() {
+ return fieldSetFlags()[7];
+ }
+
+ /** Clears the value of the 'type_array_string' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearTypeArrayString() {
+ type_array_string = null;
+ fieldSetFlags()[7] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'type_array_boolean' field */
+ public java.util.List<java.lang.Boolean> getTypeArrayBoolean() {
+ return type_array_boolean;
+ }
+
+ /** Sets the value of the 'type_array_boolean' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setTypeArrayBoolean(java.util.List<java.lang.Boolean> value) {
+ validate(fields()[8], value);
+ this.type_array_boolean = value;
+ fieldSetFlags()[8] = true;
+ return this;
+ }
+
+ /** Checks whether the 'type_array_boolean' field has been set */
+ public boolean hasTypeArrayBoolean() {
+ return fieldSetFlags()[8];
+ }
+
+ /** Clears the value of the 'type_array_boolean' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearTypeArrayBoolean() {
+ type_array_boolean = null;
+ fieldSetFlags()[8] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'type_nullable_array' field */
+ public java.util.List<java.lang.CharSequence> getTypeNullableArray() {
+ return type_nullable_array;
+ }
+
+ /** Sets the value of the 'type_nullable_array' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setTypeNullableArray(java.util.List<java.lang.CharSequence> value) {
+ validate(fields()[9], value);
+ this.type_nullable_array = value;
+ fieldSetFlags()[9] = true;
+ return this;
+ }
+
+ /** Checks whether the 'type_nullable_array' field has been set */
+ public boolean hasTypeNullableArray() {
+ return fieldSetFlags()[9];
+ }
+
+ /** Clears the value of the 'type_nullable_array' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearTypeNullableArray() {
+ type_nullable_array = null;
+ fieldSetFlags()[9] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'type_enum' field */
+ public org.apache.flink.api.io.avro.generated.Colors getTypeEnum() {
+ return type_enum;
+ }
+
+ /** Sets the value of the 'type_enum' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setTypeEnum(org.apache.flink.api.io.avro.generated.Colors value) {
+ validate(fields()[10], value);
+ this.type_enum = value;
+ fieldSetFlags()[10] = true;
+ return this;
+ }
+
+ /** Checks whether the 'type_enum' field has been set */
+ public boolean hasTypeEnum() {
+ return fieldSetFlags()[10];
+ }
+
+ /** Clears the value of the 'type_enum' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearTypeEnum() {
+ type_enum = null;
+ fieldSetFlags()[10] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'type_map' field */
+ public java.util.Map<java.lang.CharSequence,java.lang.Long> getTypeMap() {
+ return type_map;
+ }
+
+ /** Sets the value of the 'type_map' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder setTypeMap(java.util.Map<java.lang.CharSequence,java.lang.Long> value) {
+ validate(fields()[11], value);
+ this.type_map = value;
+ fieldSetFlags()[11] = true;
+ return this;
+ }
+
+ /** Checks whether the 'type_map' field has been set */
+ public boolean hasTypeMap() {
+ return fieldSetFlags()[11];
+ }
+
+ /** Clears the value of the 'type_map' field */
+ public org.apache.flink.api.io.avro.generated.User.Builder clearTypeMap() {
+ type_map = null;
+ fieldSetFlags()[11] = false;
+ return this;
+ }
+
+ @Override
+ public User build() {
+ try {
+ User record = new User();
+ record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]);
+ record.favorite_number = fieldSetFlags()[1] ? this.favorite_number : (java.lang.Integer) defaultValue(fields()[1]);
+ record.favorite_color = fieldSetFlags()[2] ? this.favorite_color : (java.lang.CharSequence) defaultValue(fields()[2]);
+ record.type_long_test = fieldSetFlags()[3] ? this.type_long_test : (java.lang.Long) defaultValue(fields()[3]);
+ record.type_double_test = fieldSetFlags()[4] ? this.type_double_test : (java.lang.Object) defaultValue(fields()[4]);
+ record.type_null_test = fieldSetFlags()[5] ? this.type_null_test : (java.lang.Object) defaultValue(fields()[5]);
+ record.type_bool_test = fieldSetFlags()[6] ? this.type_bool_test : (java.lang.Object) defaultValue(fields()[6]);
+ record.type_array_string = fieldSetFlags()[7] ? this.type_array_string : (java.util.List<java.lang.CharSequence>) defaultValue(fields()[7]);
+ record.type_array_boolean = fieldSetFlags()[8] ? this.type_array_boolean : (java.util.List<java.lang.Boolean>) defaultValue(fields()[8]);
+ record.type_nullable_array = fieldSetFlags()[9] ? this.type_nullable_array : (java.util.List<java.lang.CharSequence>) defaultValue(fields()[9]);
+ record.type_enum = fieldSetFlags()[10] ? this.type_enum : (org.apache.flink.api.io.avro.generated.Colors) defaultValue(fields()[10]);
+ record.type_map = fieldSetFlags()[11] ? this.type_map : (java.util.Map<java.lang.CharSequence,java.lang.Long>) defaultValue(fields()[11]);
+ return record;
+ } catch (Exception e) {
+ throw new org.apache.avro.AvroRuntimeException(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
new file mode 100644
index 0000000..23fbab3
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.java.io;
+
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.core.fs.Path;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class AvroInputFormatTypeExtractionTest {
+
+ @Test
+ public void testTypeExtraction() {
+ try {
+ InputFormat<MyAvroType, ?> format = new AvroInputFormat<MyAvroType>(new Path("file:///ignore/this/file"), MyAvroType.class);
+
+ TypeInformation<?> typeInfoDirect = TypeExtractor.getInputFormatTypes(format);
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<MyAvroType> input = env.createInput(format);
+ TypeInformation<?> typeInfoDataSet = input.getType();
+
+
+ Assert.assertTrue(typeInfoDirect instanceof PojoTypeInfo);
+ Assert.assertTrue(typeInfoDataSet instanceof PojoTypeInfo);
+
+ Assert.assertEquals(MyAvroType.class, typeInfoDirect.getTypeClass());
+ Assert.assertEquals(MyAvroType.class, typeInfoDataSet.getTypeClass());
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ public static final class MyAvroType {
+
+ public String theString;
+
+ private double aDouble;
+
+ public double getaDouble() {
+ return aDouble;
+ }
+
+ public void setaDouble(double aDouble) {
+ this.aDouble = aDouble;
+ }
+
+ public void setTheString(String theString) {
+ this.theString = theString;
+ }
+
+ public String getTheString() {
+ return theString;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/resources/avro/user.avsc
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/resources/avro/user.avsc b/flink-staging/flink-avro/src/test/resources/avro/user.avsc
new file mode 100644
index 0000000..af3cb75
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/resources/avro/user.avsc
@@ -0,0 +1,19 @@
+
+{"namespace": "org.apache.flink.api.java.record.io.avro.generated",
+ "type": "record",
+ "name": "User",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "favorite_number", "type": ["int", "null"]},
+ {"name": "favorite_color", "type": ["string", "null"]},
+ {"name": "type_long_test", "type": ["long", "null"]},
+ {"name": "type_double_test", "type": ["double"]},
+ {"name": "type_null_test", "type": ["null"]},
+ {"name": "type_bool_test", "type": ["boolean"]},
+ {"name": "type_array_string", "type" : {"type" : "array", "items" : "string"}},
+ {"name": "type_array_boolean", "type" : {"type" : "array", "items" : "boolean"}},
+ {"name": "type_nullable_array", "type": ["null", {"type":"array", "items":"string"}], "default":null},
+ {"name": "type_enum", "type": {"type": "enum", "name": "Colors", "symbols" : ["RED", "GREEN", "BLUE"]}},
+ {"name": "type_map", "type": {"type": "map", "values": "long"}}
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/resources/log4j-test.properties b/flink-staging/flink-avro/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..0b686e5
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/resources/log4j-test.properties
@@ -0,0 +1,27 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=OFF, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/resources/logback-test.xml b/flink-staging/flink-avro/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..8b3bb27
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/resources/logback-test.xml
@@ -0,0 +1,29 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/resources/testdata.avro
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/resources/testdata.avro b/flink-staging/flink-avro/src/test/resources/testdata.avro
new file mode 100644
index 0000000..45308b9
Binary files /dev/null and b/flink-staging/flink-avro/src/test/resources/testdata.avro differ
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/pom.xml b/flink-staging/flink-hadoop-compatibility/pom.xml
new file mode 100644
index 0000000..e39f8b6
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/pom.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-staging</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-hadoop-compatibility</artifactId>
+ <name>flink-hadoop-compatibility</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-tests</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>hadoop-2</id>
+ <activation>
+ <property>
+ <!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh -->
+ <!--hadoop2--><name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopInputFormat.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopInputFormat.java
new file mode 100644
index 0000000..326a1c4
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopInputFormat.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.ArrayList;
+
+import org.apache.flink.api.common.io.LocatableInputSplitAssigner;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
+import org.apache.flink.api.common.io.statistics.BaseStatistics;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.io.InputSplitAssigner;
+import org.apache.flink.hadoopcompatibility.mapred.utils.HadoopUtils;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopInputSplit;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.util.ReflectionUtils;
+
+public class HadoopInputFormat<K, V> implements InputFormat<Tuple2<K,V>, HadoopInputSplit>, ResultTypeQueryable<Tuple2<K,V>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(HadoopInputFormat.class);
+
+ private org.apache.hadoop.mapred.InputFormat<K, V> mapredInputFormat;
+ private Class<K> keyClass;
+ private Class<V> valueClass;
+ private JobConf jobConf;
+
+ private transient K key;
+ private transient V value;
+
+ private transient RecordReader<K, V> recordReader;
+ private transient boolean fetched = false;
+ private transient boolean hasNext;
+
+ public HadoopInputFormat() {
+ super();
+ }
+
+ public HadoopInputFormat(org.apache.hadoop.mapred.InputFormat<K,V> mapredInputFormat, Class<K> key, Class<V> value, JobConf job) {
+ super();
+ this.mapredInputFormat = mapredInputFormat;
+ this.keyClass = key;
+ this.valueClass = value;
+ HadoopUtils.mergeHadoopConf(job);
+ this.jobConf = job;
+ ReflectionUtils.setConf(mapredInputFormat, jobConf);
+ }
+
+ public void setJobConf(JobConf job) {
+ this.jobConf = job;
+ }
+
+ public org.apache.hadoop.mapred.InputFormat<K,V> getHadoopInputFormat() {
+ return mapredInputFormat;
+ }
+
+ public void setHadoopInputFormat(org.apache.hadoop.mapred.InputFormat<K,V> mapredInputFormat) {
+ this.mapredInputFormat = mapredInputFormat;
+ }
+
+ public JobConf getJobConf() {
+ return jobConf;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // InputFormat
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void configure(Configuration parameters) {
+ // nothing to do
+ }
+
+ @Override
+ public BaseStatistics getStatistics(BaseStatistics cachedStats) throws IOException {
+ // only gather base statistics for FileInputFormats
+ if(!(mapredInputFormat instanceof FileInputFormat)) {
+ return null;
+ }
+
+ final FileBaseStatistics cachedFileStats = (cachedStats != null && cachedStats instanceof FileBaseStatistics) ?
+ (FileBaseStatistics) cachedStats : null;
+
+ try {
+ final org.apache.hadoop.fs.Path[] paths = FileInputFormat.getInputPaths(this.jobConf);
+
+ return getFileStats(cachedFileStats, paths, new ArrayList<FileStatus>(1));
+ } catch (IOException ioex) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn("Could not determine statistics due to an io error: "
+ + ioex.getMessage());
+ }
+ } catch (Throwable t) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Unexpected problem while getting the file statistics: "
+ + t.getMessage(), t);
+ }
+ }
+
+ // no statistics available
+ return null;
+ }
+
+ @Override
+ public HadoopInputSplit[] createInputSplits(int minNumSplits)
+ throws IOException {
+ org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
+ HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
+ for(int i=0;i<splitArray.length;i++){
+ hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
+ }
+ return hiSplit;
+ }
+
+ @Override
+ public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
+ return new LocatableInputSplitAssigner(inputSplits);
+ }
+
+ @Override
+ public void open(HadoopInputSplit split) throws IOException {
+ this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
+ if (this.recordReader instanceof Configurable) {
+ ((Configurable) this.recordReader).setConf(jobConf);
+ }
+ key = this.recordReader.createKey();
+ value = this.recordReader.createValue();
+ this.fetched = false;
+ }
+
+ @Override
+ public boolean reachedEnd() throws IOException {
+ if(!fetched) {
+ fetchNext();
+ }
+ return !hasNext;
+ }
+
+ private void fetchNext() throws IOException {
+ hasNext = this.recordReader.next(key, value);
+ fetched = true;
+ }
+
+ @Override
+ public Tuple2<K, V> nextRecord(Tuple2<K, V> record) throws IOException {
+ if(!fetched) {
+ fetchNext();
+ }
+ if(!hasNext) {
+ return null;
+ }
+ record.f0 = key;
+ record.f1 = value;
+ fetched = false;
+ return record;
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.recordReader.close();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Helper methods
+ // --------------------------------------------------------------------------------------------
+
+ private FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, org.apache.hadoop.fs.Path[] hadoopFilePaths,
+ ArrayList<FileStatus> files) throws IOException {
+
+ long latestModTime = 0L;
+
+ // get the file info and check whether the cached statistics are still valid.
+ for(org.apache.hadoop.fs.Path hadoopPath : hadoopFilePaths) {
+
+ final Path filePath = new Path(hadoopPath.toUri());
+ final FileSystem fs = FileSystem.get(filePath.toUri());
+
+ final FileStatus file = fs.getFileStatus(filePath);
+ latestModTime = Math.max(latestModTime, file.getModificationTime());
+
+ // enumerate all files and check their modification time stamp.
+ if (file.isDir()) {
+ FileStatus[] fss = fs.listStatus(filePath);
+ files.ensureCapacity(files.size() + fss.length);
+
+ for (FileStatus s : fss) {
+ if (!s.isDir()) {
+ files.add(s);
+ latestModTime = Math.max(s.getModificationTime(), latestModTime);
+ }
+ }
+ } else {
+ files.add(file);
+ }
+ }
+
+ // check whether the cached statistics are still valid, if we have any
+ if (cachedStats != null && latestModTime <= cachedStats.getLastModificationTime()) {
+ return cachedStats;
+ }
+
+ // calculate the whole length
+ long len = 0;
+ for (FileStatus s : files) {
+ len += s.getLen();
+ }
+
+ // sanity check
+ if (len <= 0) {
+ len = BaseStatistics.SIZE_UNKNOWN;
+ }
+
+ return new FileBaseStatistics(latestModTime, len, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Custom serialization methods
+ // --------------------------------------------------------------------------------------------
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeUTF(mapredInputFormat.getClass().getName());
+ out.writeUTF(keyClass.getName());
+ out.writeUTF(valueClass.getName());
+ jobConf.write(out);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ String hadoopInputFormatClassName = in.readUTF();
+ String keyClassName = in.readUTF();
+ String valueClassName = in.readUTF();
+ if(jobConf == null) {
+ jobConf = new JobConf();
+ }
+ jobConf.readFields(in);
+ try {
+ this.mapredInputFormat = (org.apache.hadoop.mapred.InputFormat<K,V>) Class.forName(hadoopInputFormatClassName, true, Thread.currentThread().getContextClassLoader()).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to instantiate the hadoop input format", e);
+ }
+ try {
+ this.keyClass = (Class<K>) Class.forName(keyClassName, true, Thread.currentThread().getContextClassLoader());
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to find key class.", e);
+ }
+ try {
+ this.valueClass = (Class<V>) Class.forName(valueClassName, true, Thread.currentThread().getContextClassLoader());
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to find value class.", e);
+ }
+ ReflectionUtils.setConf(mapredInputFormat, jobConf);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // ResultTypeQueryable
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public TypeInformation<Tuple2<K,V>> getProducedType() {
+ return new TupleTypeInfo<Tuple2<K,V>>(TypeExtractor.createTypeInfo(keyClass), TypeExtractor.createTypeInfo(valueClass));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopMapFunction.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopMapFunction.java
new file mode 100644
index 0000000..dfe0067
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopMapFunction.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapred;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopOutputCollector;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * This wrapper maps a Hadoop Mapper (mapred API) to a Flink FlatMapFunction.
+ */
+@SuppressWarnings("rawtypes")
+public final class HadoopMapFunction<KEYIN, VALUEIN, KEYOUT, VALUEOUT>
+ extends RichFlatMapFunction<Tuple2<KEYIN,VALUEIN>, Tuple2<KEYOUT,VALUEOUT>>
+ implements ResultTypeQueryable<Tuple2<KEYOUT,VALUEOUT>>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private transient Mapper<KEYIN,VALUEIN,KEYOUT,VALUEOUT> mapper;
+ private transient JobConf jobConf;
+
+ private transient HadoopOutputCollector<KEYOUT,VALUEOUT> outputCollector;
+ private transient Reporter reporter;
+
+ /**
+ * Maps a Hadoop Mapper (mapred API) to a Flink FlatMapFunction.
+ *
+ * @param hadoopMapper The Hadoop Mapper to wrap.
+ */
+ public HadoopMapFunction(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopMapper) {
+ this(hadoopMapper, new JobConf());
+ }
+
+ /**
+ * Maps a Hadoop Mapper (mapred API) to a Flink FlatMapFunction.
+ * The Hadoop Mapper is configured with the provided JobConf.
+ *
+ * @param hadoopMapper The Hadoop Mapper to wrap.
+ * @param conf The JobConf that is used to configure the Hadoop Mapper.
+ */
+ public HadoopMapFunction(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopMapper, JobConf conf) {
+ if(hadoopMapper == null) {
+ throw new NullPointerException("Mapper may not be null.");
+ }
+ if(conf == null) {
+ throw new NullPointerException("JobConf may not be null.");
+ }
+
+ this.mapper = hadoopMapper;
+ this.jobConf = conf;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ this.mapper.configure(jobConf);
+
+ this.reporter = new HadoopDummyReporter();
+ this.outputCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
+ }
+
+ @Override
+ public void flatMap(final Tuple2<KEYIN,VALUEIN> value, final Collector<Tuple2<KEYOUT,VALUEOUT>> out)
+ throws Exception {
+ outputCollector.setFlinkCollector(out);
+ mapper.map(value.f0, value.f1, outputCollector, reporter);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public TypeInformation<Tuple2<KEYOUT,VALUEOUT>> getProducedType() {
+ Class<KEYOUT> outKeyClass = (Class<KEYOUT>) TypeExtractor.getParameterType(Mapper.class, mapper.getClass(), 2);
+ Class<VALUEOUT> outValClass = (Class<VALUEOUT>)TypeExtractor.getParameterType(Mapper.class, mapper.getClass(), 3);
+
+ final TypeInformation<KEYOUT> keyTypeInfo = TypeExtractor.getForClass((Class<KEYOUT>) outKeyClass);
+ final TypeInformation<VALUEOUT> valueTypleInfo = TypeExtractor.getForClass((Class<VALUEOUT>) outValClass);
+ return new TupleTypeInfo<Tuple2<KEYOUT,VALUEOUT>>(keyTypeInfo, valueTypleInfo);
+ }
+
+ /**
+ * Custom serialization methods.
+ * @see http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html
+ */
+ private void writeObject(final ObjectOutputStream out) throws IOException {
+ out.writeObject(mapper.getClass());
+ jobConf.write(out);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
+ Class<Mapper<KEYIN,VALUEIN,KEYOUT,VALUEOUT>> mapperClass =
+ (Class<Mapper<KEYIN,VALUEIN,KEYOUT,VALUEOUT>>)in.readObject();
+ mapper = InstantiationUtil.instantiate(mapperClass);
+
+ jobConf = new JobConf();
+ jobConf.readFields(in);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopOutputFormat.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopOutputFormat.java
new file mode 100644
index 0000000..f3abfcd
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopOutputFormat.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+import org.apache.flink.api.common.io.FinalizeOnMaster;
+import org.apache.flink.api.common.io.OutputFormat;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.hadoopcompatibility.mapred.utils.HadoopUtils;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyProgressable;
+import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
+import org.apache.hadoop.mapred.FileOutputCommitter;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+import org.apache.hadoop.mapred.TaskAttemptID;
+import org.apache.hadoop.util.ReflectionUtils;
+
+
+public class HadoopOutputFormat<K,V> implements OutputFormat<Tuple2<K,V>>, FinalizeOnMaster {
+
+ private static final long serialVersionUID = 1L;
+
+ private JobConf jobConf;
+ private org.apache.hadoop.mapred.OutputFormat<K,V> mapredOutputFormat;
+ private transient RecordWriter<K,V> recordWriter;
+ private transient FileOutputCommitter fileOutputCommitter;
+ private transient TaskAttemptContext context;
+ private transient JobContext jobContext;
+
+ public HadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K,V> mapredOutputFormat, JobConf job) {
+ super();
+ this.mapredOutputFormat = mapredOutputFormat;
+ HadoopUtils.mergeHadoopConf(job);
+ this.jobConf = job;
+ }
+
+ public void setJobConf(JobConf job) {
+ this.jobConf = job;
+ }
+
+ public JobConf getJobConf() {
+ return jobConf;
+ }
+
+ public org.apache.hadoop.mapred.OutputFormat<K,V> getHadoopOutputFormat() {
+ return mapredOutputFormat;
+ }
+
+ public void setHadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K,V> mapredOutputFormat) {
+ this.mapredOutputFormat = mapredOutputFormat;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // OutputFormat
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void configure(Configuration parameters) {
+ // nothing to do
+ }
+
+ /**
+ * create the temporary output file for hadoop RecordWriter.
+ * @param taskNumber The number of the parallel instance.
+ * @param numTasks The number of parallel tasks.
+ * @throws IOException
+ */
+ @Override
+ public void open(int taskNumber, int numTasks) throws IOException {
+ if (Integer.toString(taskNumber + 1).length() > 6) {
+ throw new IOException("Task id too large.");
+ }
+
+ TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_"
+ + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s"," ").replace(" ", "0")
+ + Integer.toString(taskNumber + 1)
+ + "_0");
+
+ this.jobConf.set("mapred.task.id", taskAttemptID.toString());
+ this.jobConf.setInt("mapred.task.partition", taskNumber + 1);
+ // for hadoop 2.2
+ this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString());
+ this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1);
+
+ try {
+ this.context = HadoopUtils.instantiateTaskAttemptContext(this.jobConf, taskAttemptID);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ this.fileOutputCommitter = new FileOutputCommitter();
+
+ try {
+ this.jobContext = HadoopUtils.instantiateJobContext(this.jobConf, new JobID());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ this.fileOutputCommitter.setupJob(jobContext);
+
+ this.recordWriter = this.mapredOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable());
+ }
+
+ @Override
+ public void writeRecord(Tuple2<K, V> record) throws IOException {
+ this.recordWriter.write(record.f0, record.f1);
+ }
+
+ /**
+ * commit the task by moving the output file out from the temporary directory.
+ * @throws IOException
+ */
+ @Override
+ public void close() throws IOException {
+ this.recordWriter.close(new HadoopDummyReporter());
+
+ if (this.fileOutputCommitter.needsTaskCommit(this.context)) {
+ this.fileOutputCommitter.commitTask(this.context);
+ }
+ }
+
+ @Override
+ public void finalizeGlobal(int parallelism) throws IOException {
+
+ try {
+ JobContext jobContext = HadoopUtils.instantiateJobContext(this.jobConf, new JobID());
+ FileOutputCommitter fileOutputCommitter = new FileOutputCommitter();
+
+ // finalize HDFS output format
+ fileOutputCommitter.commitJob(jobContext);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Custom serialization methods
+ // --------------------------------------------------------------------------------------------
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeUTF(mapredOutputFormat.getClass().getName());
+ jobConf.write(out);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ String hadoopOutputFormatName = in.readUTF();
+ if(jobConf == null) {
+ jobConf = new JobConf();
+ }
+ jobConf.readFields(in);
+ try {
+ this.mapredOutputFormat = (org.apache.hadoop.mapred.OutputFormat<K,V>) Class.forName(hadoopOutputFormatName, true, Thread.currentThread().getContextClassLoader()).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to instantiate the hadoop output format", e);
+ }
+ ReflectionUtils.setConf(mapredOutputFormat, jobConf);
+ }
+}
[21/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormat.java b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormat.java
new file mode 100644
index 0000000..3cd295b
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormat.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.record.io.jdbc;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.flink.api.common.io.NonParallelInput;
+import org.apache.flink.api.java.record.io.GenericInputFormat;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.types.BooleanValue;
+import org.apache.flink.types.DoubleValue;
+import org.apache.flink.types.FloatValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.ShortValue;
+import org.apache.flink.types.StringValue;
+
+/**
+ * InputFormat to read data from a database and generate PactReords.
+ * The InputFormat has to be configured with the query, and either all
+ * connection parameters or a complete database URL.{@link Configuration} The position of a value inside a Record is
+ * determined by the table
+ * returned.
+ *
+ * @see Configuration
+ * @see Record
+ * @see DriverManager
+ */
+public class JDBCInputFormat extends GenericInputFormat implements NonParallelInput {
+
+ private static final long serialVersionUID = 1L;
+
+ @SuppressWarnings("unused")
+ private static final Logger LOG = LoggerFactory.getLogger(JDBCInputFormat.class);
+
+
+ public final String DRIVER_KEY = "driver";
+ public final String USERNAME_KEY = "username";
+ public final String PASSWORD_KEY = "password";
+ public final String URL_KEY = "url";
+ public final String QUERY_KEY = "query";
+
+
+ private String username;
+ private String password;
+ private String driverName;
+ private String dbURL;
+ private String query;
+
+
+ private transient Connection dbConn;
+ private transient Statement statement;
+ private transient ResultSet resultSet;
+
+
+ /**
+ * Creates a non-configured JDBCInputFormat. This format has to be
+ * configured using configure(configuration).
+ */
+ public JDBCInputFormat() {}
+
+ /**
+ * Creates a JDBCInputFormat and configures it.
+ *
+ * @param driverName
+ * JDBC-Drivename
+ * @param dbURL
+ * Formatted URL containing all connection parameters.
+ * @param username
+ * @param password
+ * @param query
+ * Query to execute.
+ */
+ public JDBCInputFormat(String driverName, String dbURL, String username, String password, String query) {
+ this.driverName = driverName;
+ this.query = query;
+ this.dbURL = dbURL;
+ this.username = username;
+ this.password = password;
+ }
+
+ /**
+ * Creates a JDBCInputFormat and configures it.
+ *
+ * @param driverName
+ * JDBC-Drivername
+ * @param dbURL
+ * Formatted URL containing all connection parameters.
+ * @param query
+ * Query to execute.
+ */
+ public JDBCInputFormat(String driverName, String dbURL, String query) {
+ this(driverName, dbURL, "", "", query);
+ }
+
+ /**
+ * Creates a JDBCInputFormat and configures it.
+ *
+ * @param parameters
+ * Configuration with all connection parameters.
+ * @param query
+ * Query to execute.
+ */
+ public JDBCInputFormat(Configuration parameters, String query) {
+ this.driverName = parameters.getString(DRIVER_KEY, "");
+ this.username = parameters.getString(USERNAME_KEY, "");
+ this.password = parameters.getString(PASSWORD_KEY, "");
+ this.dbURL = parameters.getString(URL_KEY, "");
+ this.query = query;
+ }
+
+
+ /**
+ * Configures this JDBCInputFormat. This includes setting the connection
+ * parameters (if necessary), establishing the connection and executing the
+ * query.
+ *
+ * @param parameters
+ * Configuration containing all or no parameters.
+ */
+ @Override
+ public void configure(Configuration parameters) {
+ boolean needConfigure = isFieldNullOrEmpty(this.query) || isFieldNullOrEmpty(this.dbURL);
+ if (needConfigure) {
+ this.driverName = parameters.getString(DRIVER_KEY, null);
+ this.username = parameters.getString(USERNAME_KEY, null);
+ this.password = parameters.getString(PASSWORD_KEY, null);
+ this.query = parameters.getString(QUERY_KEY, null);
+ this.dbURL = parameters.getString(URL_KEY, null);
+ }
+
+ try {
+ prepareQueryExecution();
+ } catch (SQLException e) {
+ throw new IllegalArgumentException("Configure failed:\t!", e);
+ }
+ }
+
+ /**
+ * Enters data value from the current resultSet into a Record.
+ *
+ * @param pos
+ * Record position to be set.
+ * @param type
+ * SQL type of the resultSet value.
+ * @param record
+ * Target Record.
+ */
+ private void retrieveTypeAndFillRecord(int pos, int type, Record record) throws SQLException,
+ NotTransformableSQLFieldException {
+ switch (type) {
+ case java.sql.Types.NULL:
+ record.setField(pos, NullValue.getInstance());
+ break;
+ case java.sql.Types.BOOLEAN:
+ record.setField(pos, new BooleanValue(resultSet.getBoolean(pos + 1)));
+ break;
+ case java.sql.Types.BIT:
+ record.setField(pos, new BooleanValue(resultSet.getBoolean(pos + 1)));
+ break;
+ case java.sql.Types.CHAR:
+ record.setField(pos, new StringValue(resultSet.getString(pos + 1)));
+ break;
+ case java.sql.Types.NCHAR:
+ record.setField(pos, new StringValue(resultSet.getString(pos + 1)));
+ break;
+ case java.sql.Types.VARCHAR:
+ record.setField(pos, new StringValue(resultSet.getString(pos + 1)));
+ break;
+ case java.sql.Types.LONGVARCHAR:
+ record.setField(pos, new StringValue(resultSet.getString(pos + 1)));
+ break;
+ case java.sql.Types.LONGNVARCHAR:
+ record.setField(pos, new StringValue(resultSet.getString(pos + 1)));
+ break;
+ case java.sql.Types.TINYINT:
+ record.setField(pos, new ShortValue(resultSet.getShort(pos + 1)));
+ break;
+ case java.sql.Types.SMALLINT:
+ record.setField(pos, new ShortValue(resultSet.getShort(pos + 1)));
+ break;
+ case java.sql.Types.BIGINT:
+ record.setField(pos, new LongValue(resultSet.getLong(pos + 1)));
+ break;
+ case java.sql.Types.INTEGER:
+ record.setField(pos, new IntValue(resultSet.getInt(pos + 1)));
+ break;
+ case java.sql.Types.FLOAT:
+ record.setField(pos, new DoubleValue(resultSet.getDouble(pos + 1)));
+ break;
+ case java.sql.Types.REAL:
+ record.setField(pos, new FloatValue(resultSet.getFloat(pos + 1)));
+ break;
+ case java.sql.Types.DOUBLE:
+ record.setField(pos, new DoubleValue(resultSet.getDouble(pos + 1)));
+ break;
+ case java.sql.Types.DECIMAL:
+ record.setField(pos, new DoubleValue(resultSet.getBigDecimal(pos + 1).doubleValue()));
+ break;
+ case java.sql.Types.NUMERIC:
+ record.setField(pos, new DoubleValue(resultSet.getBigDecimal(pos + 1).doubleValue()));
+ break;
+ case java.sql.Types.DATE:
+ record.setField(pos, new StringValue(resultSet.getDate(pos + 1).toString()));
+ break;
+ case java.sql.Types.TIME:
+ record.setField(pos, new LongValue(resultSet.getTime(pos + 1).getTime()));
+ break;
+ case java.sql.Types.TIMESTAMP:
+ record.setField(pos, new StringValue(resultSet.getTimestamp(pos + 1).toString()));
+ break;
+ case java.sql.Types.SQLXML:
+ record.setField(pos, new StringValue(resultSet.getSQLXML(pos + 1).toString()));
+ break;
+ default:
+ throw new NotTransformableSQLFieldException("Unknown sql-type [" + type + "]on column [" + pos + "]");
+
+ // case java.sql.Types.BINARY:
+ // case java.sql.Types.VARBINARY:
+ // case java.sql.Types.LONGVARBINARY:
+ // case java.sql.Types.ARRAY:
+ // case java.sql.Types.JAVA_OBJECT:
+ // case java.sql.Types.BLOB:
+ // case java.sql.Types.CLOB:
+ // case java.sql.Types.NCLOB:
+ // case java.sql.Types.DATALINK:
+ // case java.sql.Types.DISTINCT:
+ // case java.sql.Types.OTHER:
+ // case java.sql.Types.REF:
+ // case java.sql.Types.ROWID:
+ // case java.sql.Types.STRUCT:
+ }
+ }
+
+ private boolean isFieldNullOrEmpty(String field) {
+ return (field == null || field.length() == 0);
+ }
+
+ private void prepareQueryExecution() throws SQLException {
+ setClassForDBType();
+ prepareCredentialsAndExecute();
+ }
+
+ /**
+ * Loads appropriate JDBC driver.
+ *
+ * @param dbType
+ * Type of the database.
+ * @return boolean value, indication whether an appropriate driver could be
+ * found.
+ */
+ private void setClassForDBType() {
+ try {
+ Class.forName(driverName);
+ } catch (ClassNotFoundException cnfe) {
+ throw new IllegalArgumentException("JDBC-Class not found:\t" + cnfe.getLocalizedMessage());
+ }
+ }
+
+ private void prepareCredentialsAndExecute() throws SQLException {
+ if (isFieldNullOrEmpty(username)) {
+ prepareConnection(dbURL);
+ } else {
+ prepareConnection();
+ }
+ executeQuery();
+ }
+
+ /**
+ * Establishes a connection to a database.
+ *
+ * @param dbURL
+ * Assembled URL containing all connection parameters.
+ * @return boolean value, indicating whether a connection could be
+ * established
+ */
+ private void prepareConnection(String dbURL) throws SQLException {
+ dbConn = DriverManager.getConnection(dbURL);
+ }
+
+ /**
+ * Assembles the Database URL and establishes a connection.
+ *
+ * @param dbType
+ * Type of the database.
+ * @param username
+ * Login username.
+ * @param password
+ * Login password.
+ * @return boolean value, indicating whether a connection could be
+ * established
+ */
+ private void prepareConnection() throws SQLException {
+ dbConn = DriverManager.getConnection(dbURL, username, password);
+ }
+
+ private void executeQuery() throws SQLException {
+ statement = dbConn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
+ resultSet = statement.executeQuery(this.query);
+ }
+
+ /**
+ * Checks whether all data has been read.
+ *
+ * @return boolean value indication whether all data has been read.
+ */
+ @Override
+ public boolean reachedEnd() {
+ try {
+ if (resultSet.isLast()) {
+ resultSet.close();
+ statement.close();
+ dbConn.close();
+ return true;
+ } else {
+ return false;
+ }
+ } catch (SQLException e) {
+ throw new IllegalArgumentException("Couldn't evaluate reachedEnd():\t" + e.getMessage());
+ } catch (NullPointerException e) {
+ throw new IllegalArgumentException("Couldn't access resultSet:\t" + e.getMessage());
+ }
+ }
+
+ /**
+ * Stores the next resultSet row in a Record
+ *
+ * @param record
+ * target Record
+ * @return boolean value indicating that the operation was successful
+ */
+ @Override
+ public Record nextRecord(Record record) {
+ try {
+ resultSet.next();
+ ResultSetMetaData rsmd = resultSet.getMetaData();
+ int column_count = rsmd.getColumnCount();
+ record.setNumFields(column_count);
+
+ for (int pos = 0; pos < column_count; pos++) {
+ int type = rsmd.getColumnType(pos + 1);
+ retrieveTypeAndFillRecord(pos, type, record);
+ }
+ return record;
+ } catch (SQLException e) {
+ throw new IllegalArgumentException("Couldn't read data:\t" + e.getMessage());
+ } catch (NotTransformableSQLFieldException e) {
+ throw new IllegalArgumentException("Couldn't read data because of unknown column sql-type:\t"
+ + e.getMessage());
+ } catch (NullPointerException e) {
+ throw new IllegalArgumentException("Couldn't access resultSet:\t" + e.getMessage());
+ }
+ }
+
+ public static class NotTransformableSQLFieldException extends Exception {
+
+ private static final long serialVersionUID = 1L;
+
+ public NotTransformableSQLFieldException(String message) {
+ super(message);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormat.java b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormat.java
new file mode 100644
index 0000000..780001a
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormat.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.record.io.jdbc;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+import org.apache.flink.api.common.io.FileOutputFormat;
+import org.apache.flink.api.common.io.OutputFormat;
+import org.apache.flink.api.java.record.operators.GenericDataSink;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.types.BooleanValue;
+import org.apache.flink.types.ByteValue;
+import org.apache.flink.types.CharValue;
+import org.apache.flink.types.DoubleValue;
+import org.apache.flink.types.FloatValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.ShortValue;
+import org.apache.flink.types.StringValue;
+import org.apache.flink.types.Value;
+
+public class JDBCOutputFormat implements OutputFormat<Record> {
+ private static final long serialVersionUID = 1L;
+
+ private static final int DEFAULT_BATCH_INTERVERAL = 5000;
+
+ public static final String DRIVER_KEY = "driver";
+ public static final String USERNAME_KEY = "username";
+ public static final String PASSWORD_KEY = "password";
+ public static final String URL_KEY = "url";
+ public static final String QUERY_KEY = "query";
+ public static final String FIELD_COUNT_KEY = "fields";
+ public static final String FIELD_TYPE_KEY = "type";
+ public static final String BATCH_INTERVAL = "batchInt";
+
+ private Connection dbConn;
+ private PreparedStatement upload;
+
+ private String username;
+ private String password;
+ private String driverName;
+ private String dbURL;
+
+ private String query;
+ private int fieldCount;
+ private Class<? extends Value>[] fieldClasses;
+
+ /**
+ * Variable indicating the current number of insert sets in a batch.
+ */
+ private int batchCount = 0;
+
+ /**
+ * Commit interval of batches.
+ * High batch interval: faster inserts, more memory required (reduce if OutOfMemoryExceptions occur)
+ * low batch interval: slower inserts, less memory.
+ */
+ private int batchInterval = DEFAULT_BATCH_INTERVERAL;
+
+
+ /**
+ * Configures this JDBCOutputFormat.
+ *
+ * @param parameters
+ * Configuration containing all parameters.
+ */
+ @Override
+ public void configure(Configuration parameters) {
+ this.driverName = parameters.getString(DRIVER_KEY, null);
+ this.username = parameters.getString(USERNAME_KEY, null);
+ this.password = parameters.getString(PASSWORD_KEY, null);
+ this.dbURL = parameters.getString(URL_KEY, null);
+ this.query = parameters.getString(QUERY_KEY, null);
+ this.fieldCount = parameters.getInteger(FIELD_COUNT_KEY, 0);
+ this.batchInterval = parameters.getInteger(BATCH_INTERVAL, DEFAULT_BATCH_INTERVERAL);
+
+ @SuppressWarnings("unchecked")
+ Class<Value>[] classes = new Class[this.fieldCount];
+ this.fieldClasses = classes;
+
+ ClassLoader cl = getClass().getClassLoader();
+
+ try {
+ for (int i = 0; i < this.fieldCount; i++) {
+ Class<? extends Value> clazz = parameters.<Value>getClass(FIELD_TYPE_KEY + i, null, cl);
+ if (clazz == null) {
+ throw new IllegalArgumentException("Invalid configuration for JDBCOutputFormat: "
+ + "No type class for parameter " + i);
+ }
+ this.fieldClasses[i] = clazz;
+ }
+ }
+ catch (ClassNotFoundException e) {
+ throw new RuntimeException("Could not load data type classes.", e);
+ }
+ }
+
+ /**
+ * Connects to the target database and initializes the prepared statement.
+ *
+ * @param taskNumber The number of the parallel instance.
+ * @throws IOException Thrown, if the output could not be opened due to an
+ * I/O problem.
+ */
+ @Override
+ public void open(int taskNumber, int numTasks) throws IOException {
+ try {
+ establishConnection();
+ upload = dbConn.prepareStatement(query);
+ } catch (SQLException sqe) {
+ throw new IllegalArgumentException("open() failed:\t!", sqe);
+ } catch (ClassNotFoundException cnfe) {
+ throw new IllegalArgumentException("JDBC-Class not found:\t", cnfe);
+ }
+ }
+
+ private void establishConnection() throws SQLException, ClassNotFoundException {
+ Class.forName(driverName);
+ if (username == null) {
+ dbConn = DriverManager.getConnection(dbURL);
+ } else {
+ dbConn = DriverManager.getConnection(dbURL, username, password);
+ }
+ }
+
+ /**
+ * Adds a record to the prepared statement.
+ * <p>
+ * When this method is called, the output format is guaranteed to be opened.
+ *
+ * @param record The records to add to the output.
+ * @throws IOException Thrown, if the records could not be added due to an
+ * I/O problem.
+ */
+
+ @Override
+ public void writeRecord(Record record) throws IOException {
+ try {
+ for (int x = 0; x < record.getNumFields(); x++) {
+ Value temp = record.getField(x, fieldClasses[x]);
+ addValue(x + 1, temp);
+ }
+ upload.addBatch();
+ batchCount++;
+ if(batchCount >= batchInterval) {
+ upload.executeBatch();
+ batchCount = 0;
+ }
+ } catch (SQLException sqe) {
+ throw new IllegalArgumentException("writeRecord() failed:\t", sqe);
+ } catch (IllegalArgumentException iae) {
+ throw new IllegalArgumentException("writeRecord() failed:\t", iae);
+ }
+ }
+
+ private enum pactType {
+ BooleanValue,
+ ByteValue,
+ CharValue,
+ DoubleValue,
+ FloatValue,
+ IntValue,
+ LongValue,
+ ShortValue,
+ StringValue
+ }
+
+ private void addValue(int index, Value value) throws SQLException {
+ pactType type;
+ try {
+ type = pactType.valueOf(value.getClass().getSimpleName());
+ } catch (IllegalArgumentException iae) {
+ throw new IllegalArgumentException("PactType not supported:\t", iae);
+ }
+ switch (type) {
+ case BooleanValue:
+ upload.setBoolean(index, ((BooleanValue) value).getValue());
+ break;
+ case ByteValue:
+ upload.setByte(index, ((ByteValue) value).getValue());
+ break;
+ case CharValue:
+ upload.setString(index, String.valueOf(((CharValue) value).getValue()));
+ break;
+ case DoubleValue:
+ upload.setDouble(index, ((DoubleValue) value).getValue());
+ break;
+ case FloatValue:
+ upload.setFloat(index, ((FloatValue) value).getValue());
+ break;
+ case IntValue:
+ upload.setInt(index, ((IntValue) value).getValue());
+ break;
+ case LongValue:
+ upload.setLong(index, ((LongValue) value).getValue());
+ break;
+ case ShortValue:
+ upload.setShort(index, ((ShortValue) value).getValue());
+ break;
+ case StringValue:
+ upload.setString(index, ((StringValue) value).getValue());
+ break;
+ }
+ }
+
+ /**
+ * Executes prepared statement and closes all resources of this instance.
+ *
+ * @throws IOException Thrown, if the input could not be closed properly.
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ upload.executeBatch();
+ batchCount = 0;
+ upload.close();
+ dbConn.close();
+ } catch (SQLException sqe) {
+ throw new IllegalArgumentException("close() failed:\t", sqe);
+ }
+ }
+
+ /**
+ * Creates a configuration builder that can be used to set the
+ * output format's parameters to the config in a fluent fashion.
+ *
+ * @return A config builder for setting parameters.
+ */
+ public static ConfigBuilder configureOutputFormat(GenericDataSink target) {
+ return new ConfigBuilder(target.getParameters());
+ }
+
+ /**
+ * Abstract builder used to set parameters to the output format's
+ * configuration in a fluent way.
+ */
+ protected static abstract class AbstractConfigBuilder<T>
+ extends FileOutputFormat.AbstractConfigBuilder<T> {
+
+ /**
+ * Creates a new builder for the given configuration.
+ *
+ * @param config The configuration into which the parameters will be written.
+ */
+ protected AbstractConfigBuilder(Configuration config) {
+ super(config);
+ }
+
+ /**
+ * Sets the query field.
+ * @param value value to be set.
+ * @return The builder itself.
+ */
+ public T setQuery(String value) {
+ this.config.setString(QUERY_KEY, value);
+ @SuppressWarnings("unchecked")
+ T ret = (T) this;
+ return ret;
+ }
+
+ /**
+ * Sets the url field.
+ * @param value value to be set.
+ * @return The builder itself.
+ */
+ public T setUrl(String value) {
+ this.config.setString(URL_KEY, value);
+ @SuppressWarnings("unchecked")
+ T ret = (T) this;
+ return ret;
+ }
+
+ /**
+ * Sets the username field.
+ * @param value value to be set.
+ * @return The builder itself.
+ */
+ public T setUsername(String value) {
+ this.config.setString(USERNAME_KEY, value);
+ @SuppressWarnings("unchecked")
+ T ret = (T) this;
+ return ret;
+ }
+
+ /**
+ * Sets the password field.
+ * @param value value to be set.
+ * @return The builder itself.
+ */
+ public T setPassword(String value) {
+ this.config.setString(PASSWORD_KEY, value);
+ @SuppressWarnings("unchecked")
+ T ret = (T) this;
+ return ret;
+ }
+
+ /**
+ * Sets the driver field.
+ * @param value value to be set.
+ * @return The builder itself.
+ */
+ public T setDriver(String value) {
+ this.config.setString(DRIVER_KEY, value);
+ @SuppressWarnings("unchecked")
+ T ret = (T) this;
+ return ret;
+ }
+
+ /**
+ * Sets the type of a column.
+ * Types are applied in the order they were set.
+ * @param type PactType to apply.
+ * @return The builder itself.
+ */
+ public T setClass(Class<? extends Value> type) {
+ final int numYet = this.config.getInteger(FIELD_COUNT_KEY, 0);
+ this.config.setClass(FIELD_TYPE_KEY + numYet, type);
+ this.config.setInteger(FIELD_COUNT_KEY, numYet + 1);
+ @SuppressWarnings("unchecked")
+ T ret = (T) this;
+ return ret;
+ }
+ }
+
+ /**
+ * A builder used to set parameters to the output format's configuration in a fluent way.
+ */
+ public static final class ConfigBuilder extends AbstractConfigBuilder<ConfigBuilder> {
+ /**
+ * Creates a new builder for the given configuration.
+ *
+ * @param targetConfig The configuration into which the parameters will be written.
+ */
+ protected ConfigBuilder(Configuration targetConfig) {
+ super(targetConfig);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/example/JDBCExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/example/JDBCExample.java b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/example/JDBCExample.java
new file mode 100644
index 0000000..58aaf9e
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/main/java/org/apache/flink/api/java/record/io/jdbc/example/JDBCExample.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.record.io.jdbc.example;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.Program;
+import org.apache.flink.api.common.ProgramDescription;
+import org.apache.flink.api.java.record.io.jdbc.JDBCInputFormat;
+import org.apache.flink.api.java.record.io.jdbc.JDBCOutputFormat;
+import org.apache.flink.api.java.record.operators.GenericDataSink;
+import org.apache.flink.api.java.record.operators.GenericDataSource;
+import org.apache.flink.client.LocalExecutor;
+import org.apache.flink.types.FloatValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.StringValue;
+
+/**
+ * Stand-alone example for the JDBC connector.
+ *
+ * NOTE: To run this example, you need the apache derby code in your classpath.
+ * See the Maven file (pom.xml) for a reference to the derby dependency. You can
+ * simply Change the scope of the Maven dependency from test to compile.
+ */
+public class JDBCExample implements Program, ProgramDescription {
+
+ @Override
+ public Plan getPlan(String[] args) {
+ /*
+ * In this example we use the constructor where the url contains all the settings that are needed.
+ * You could also use the default constructor and deliver a Configuration with all the needed settings.
+ * You also could set the settings to the source-instance.
+ */
+ GenericDataSource<JDBCInputFormat> source = new GenericDataSource<JDBCInputFormat>(
+ new JDBCInputFormat(
+ "org.apache.derby.jdbc.EmbeddedDriver",
+ "jdbc:derby:memory:ebookshop",
+ "select * from books"),
+ "Data Source");
+
+ GenericDataSink sink = new GenericDataSink(new JDBCOutputFormat(), "Data Output");
+ JDBCOutputFormat.configureOutputFormat(sink)
+ .setDriver("org.apache.derby.jdbc.EmbeddedDriver")
+ .setUrl("jdbc:derby:memory:ebookshop")
+ .setQuery("insert into newbooks (id,title,author,price,qty) values (?,?,?,?,?)")
+ .setClass(IntValue.class)
+ .setClass(StringValue.class)
+ .setClass(StringValue.class)
+ .setClass(FloatValue.class)
+ .setClass(IntValue.class);
+
+ sink.addInput(source);
+ return new Plan(sink, "JDBC Example Job");
+ }
+
+ @Override
+ public String getDescription() {
+ return "Parameter:";
+ }
+
+ /*
+ * To run this example, you need the apache derby code in your classpath!
+ */
+ public static void main(String[] args) throws Exception {
+
+ prepareTestDb();
+ JDBCExample tut = new JDBCExample();
+ JobExecutionResult res = LocalExecutor.execute(tut, args);
+ System.out.println("runtime: " + res.getNetRuntime());
+
+ System.exit(0);
+ }
+
+ private static void prepareTestDb() throws Exception {
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ Connection conn = DriverManager.getConnection(dbURL);
+
+ StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE books (");
+ sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
+ sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
+ sqlQueryBuilder.append("qty INT DEFAULT NULL,");
+ sqlQueryBuilder.append("PRIMARY KEY (id))");
+
+ Statement stat = conn.createStatement();
+ stat.executeUpdate(sqlQueryBuilder.toString());
+ stat.close();
+
+ sqlQueryBuilder = new StringBuilder("CREATE TABLE newbooks (");
+ sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
+ sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
+ sqlQueryBuilder.append("qty INT DEFAULT NULL,");
+ sqlQueryBuilder.append("PRIMARY KEY (id))");
+
+ stat = conn.createStatement();
+ stat.executeUpdate(sqlQueryBuilder.toString());
+ stat.close();
+
+ sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
+ sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
+ sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
+ sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
+ sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
+ sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
+
+ stat = conn.createStatement();
+ stat.execute(sqlQueryBuilder.toString());
+ stat.close();
+
+ conn.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java b/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
new file mode 100644
index 0000000..b76f8b8
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.io.jdbc;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.junit.Assert;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class JDBCInputFormatTest {
+ JDBCInputFormat jdbcInputFormat;
+
+ static Connection conn;
+
+ static final Object[][] dbData = {
+ {1001, ("Java for dummies"), ("Tan Ah Teck"), 11.11, 11},
+ {1002, ("More Java for dummies"), ("Tan Ah Teck"), 22.22, 22},
+ {1003, ("More Java for more dummies"), ("Mohammad Ali"), 33.33, 33},
+ {1004, ("A Cup of Java"), ("Kumar"), 44.44, 44},
+ {1005, ("A Teaspoon of Java"), ("Kevin Jones"), 55.55, 55}};
+
+ @BeforeClass
+ public static void setUpClass() {
+ try {
+ prepareDerbyDatabase();
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+ private static void prepareDerbyDatabase() throws ClassNotFoundException, SQLException {
+ System.setProperty("derby.stream.error.field", "org.apache.flink.api.java.record.io.jdbc.DevNullLogStream.DEV_NULL");
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ conn = DriverManager.getConnection(dbURL);
+ createTable();
+ insertDataToSQLTable();
+ conn.close();
+ }
+
+ private static void createTable() throws SQLException {
+ StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE books (");
+ sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
+ sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
+ sqlQueryBuilder.append("qty INT DEFAULT NULL,");
+ sqlQueryBuilder.append("PRIMARY KEY (id))");
+
+ Statement stat = conn.createStatement();
+ stat.executeUpdate(sqlQueryBuilder.toString());
+ stat.close();
+ }
+
+ private static void insertDataToSQLTable() throws SQLException {
+ StringBuilder sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
+ sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
+ sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
+ sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
+ sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
+ sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
+
+ Statement stat = conn.createStatement();
+ stat.execute(sqlQueryBuilder.toString());
+ stat.close();
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ cleanUpDerbyDatabases();
+ }
+
+ private static void cleanUpDerbyDatabases() {
+ try {
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+
+ conn = DriverManager.getConnection(dbURL);
+ Statement stat = conn.createStatement();
+ stat.executeUpdate("DROP TABLE books");
+ stat.close();
+ conn.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @After
+ public void tearDown() {
+ jdbcInputFormat = null;
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidDriver() throws IOException {
+ jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername("org.apache.derby.jdbc.idontexist")
+ .setDBUrl("jdbc:derby:memory:ebookshop")
+ .setQuery("select * from books")
+ .finish();
+ jdbcInputFormat.open(null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidURL() throws IOException {
+ jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setDBUrl("jdbc:der:iamanerror:mory:ebookshop")
+ .setQuery("select * from books")
+ .finish();
+ jdbcInputFormat.open(null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidQuery() throws IOException {
+ jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setDBUrl("jdbc:derby:memory:ebookshop")
+ .setQuery("iamnotsql")
+ .finish();
+ jdbcInputFormat.open(null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIncompleteConfiguration() throws IOException {
+ jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setQuery("select * from books")
+ .finish();
+ }
+
+ @Test(expected = IOException.class)
+ public void testIncompatibleTuple() throws IOException {
+ jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setDBUrl("jdbc:derby:memory:ebookshop")
+ .setQuery("select * from books")
+ .finish();
+ jdbcInputFormat.open(null);
+ jdbcInputFormat.nextRecord(new Tuple2());
+ }
+
+ @Test
+ public void testJDBCInputFormat() throws IOException {
+ jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setDBUrl("jdbc:derby:memory:ebookshop")
+ .setQuery("select * from books")
+ .finish();
+ jdbcInputFormat.open(null);
+ Tuple5 tuple = new Tuple5();
+ int recordCount = 0;
+ while (!jdbcInputFormat.reachedEnd()) {
+ jdbcInputFormat.nextRecord(tuple);
+ Assert.assertEquals("Field 0 should be int", Integer.class, tuple.getField(0).getClass());
+ Assert.assertEquals("Field 1 should be String", String.class, tuple.getField(1).getClass());
+ Assert.assertEquals("Field 2 should be String", String.class, tuple.getField(2).getClass());
+ Assert.assertEquals("Field 3 should be float", Double.class, tuple.getField(3).getClass());
+ Assert.assertEquals("Field 4 should be int", Integer.class, tuple.getField(4).getClass());
+
+ for (int x = 0; x < 5; x++) {
+ Assert.assertEquals(dbData[recordCount][x], tuple.getField(x));
+ }
+ recordCount++;
+ }
+ Assert.assertEquals(5, recordCount);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java b/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
new file mode 100644
index 0000000..7d004f9
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.io.jdbc;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.junit.Assert;
+
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class JDBCOutputFormatTest {
+ private JDBCInputFormat jdbcInputFormat;
+ private JDBCOutputFormat jdbcOutputFormat;
+
+ private static Connection conn;
+
+ static final Object[][] dbData = {
+ {1001, ("Java for dummies"), ("Tan Ah Teck"), 11.11, 11},
+ {1002, ("More Java for dummies"), ("Tan Ah Teck"), 22.22, 22},
+ {1003, ("More Java for more dummies"), ("Mohammad Ali"), 33.33, 33},
+ {1004, ("A Cup of Java"), ("Kumar"), 44.44, 44},
+ {1005, ("A Teaspoon of Java"), ("Kevin Jones"), 55.55, 55}};
+
+ @BeforeClass
+ public static void setUpClass() throws SQLException {
+ try {
+ System.setProperty("derby.stream.error.field", "org.apache.flink.api.java.record.io.jdbc.DevNullLogStream.DEV_NULL");
+ prepareDerbyDatabase();
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ private static void prepareDerbyDatabase() throws ClassNotFoundException, SQLException {
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ conn = DriverManager.getConnection(dbURL);
+ createTable("books");
+ createTable("newbooks");
+ insertDataToSQLTables();
+ conn.close();
+ }
+
+ private static void createTable(String tableName) throws SQLException {
+ StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE ");
+ sqlQueryBuilder.append(tableName);
+ sqlQueryBuilder.append(" (");
+ sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
+ sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
+ sqlQueryBuilder.append("qty INT DEFAULT NULL,");
+ sqlQueryBuilder.append("PRIMARY KEY (id))");
+
+ Statement stat = conn.createStatement();
+ stat.executeUpdate(sqlQueryBuilder.toString());
+ stat.close();
+ }
+
+ private static void insertDataToSQLTables() throws SQLException {
+ StringBuilder sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
+ sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
+ sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
+ sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
+ sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
+ sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
+
+ Statement stat = conn.createStatement();
+ stat.execute(sqlQueryBuilder.toString());
+ stat.close();
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ cleanUpDerbyDatabases();
+ }
+
+ private static void cleanUpDerbyDatabases() {
+ try {
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+
+ conn = DriverManager.getConnection(dbURL);
+ Statement stat = conn.createStatement();
+ stat.executeUpdate("DROP TABLE books");
+ stat.executeUpdate("DROP TABLE newbooks");
+ stat.close();
+ conn.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @After
+ public void tearDown() {
+ jdbcOutputFormat = null;
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidDriver() throws IOException {
+ jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
+ .setDrivername("org.apache.derby.jdbc.idontexist")
+ .setDBUrl("jdbc:derby:memory:ebookshop")
+ .setQuery("insert into books (id, title, author, price, qty) values (?,?,?,?,?)")
+ .finish();
+ jdbcOutputFormat.open(0, 1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidURL() throws IOException {
+ jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setDBUrl("jdbc:der:iamanerror:mory:ebookshop")
+ .setQuery("insert into books (id, title, author, price, qty) values (?,?,?,?,?)")
+ .finish();
+ jdbcOutputFormat.open(0, 1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidQuery() throws IOException {
+ jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setDBUrl("jdbc:derby:memory:ebookshop")
+ .setQuery("iamnotsql")
+ .finish();
+ jdbcOutputFormat.open(0, 1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIncompleteConfiguration() throws IOException {
+ jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setQuery("insert into books (id, title, author, price, qty) values (?,?,?,?,?)")
+ .finish();
+ }
+
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIncompatibleTypes() throws IOException {
+ jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
+ .setDrivername("org.apache.derby.jdbc.EmbeddedDriver")
+ .setDBUrl("jdbc:derby:memory:ebookshop")
+ .setQuery("insert into books (id, title, author, price, qty) values (?,?,?,?,?)")
+ .finish();
+ jdbcOutputFormat.open(0, 1);
+
+ Tuple5 tuple5 = new Tuple5();
+ tuple5.setField(4, 0);
+ tuple5.setField("hello", 1);
+ tuple5.setField("world", 2);
+ tuple5.setField(0.99, 3);
+ tuple5.setField("imthewrongtype", 4);
+
+ jdbcOutputFormat.writeRecord(tuple5);
+ jdbcOutputFormat.close();
+ }
+
+ @Test
+ public void testJDBCOutputFormat() throws IOException {
+ String sourceTable = "books";
+ String targetTable = "newbooks";
+ String driverPath = "org.apache.derby.jdbc.EmbeddedDriver";
+ String dbUrl = "jdbc:derby:memory:ebookshop";
+
+ jdbcOutputFormat = JDBCOutputFormat.buildJDBCOutputFormat()
+ .setDBUrl(dbUrl)
+ .setDrivername(driverPath)
+ .setQuery("insert into " + targetTable + " (id, title, author, price, qty) values (?,?,?,?,?)")
+ .finish();
+ jdbcOutputFormat.open(0, 1);
+
+ jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername(driverPath)
+ .setDBUrl(dbUrl)
+ .setQuery("select * from " + sourceTable)
+ .finish();
+ jdbcInputFormat.open(null);
+
+ Tuple5 tuple = new Tuple5();
+ while (!jdbcInputFormat.reachedEnd()) {
+ jdbcInputFormat.nextRecord(tuple);
+ jdbcOutputFormat.writeRecord(tuple);
+ }
+
+ jdbcOutputFormat.close();
+ jdbcInputFormat.close();
+
+ jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
+ .setDrivername(driverPath)
+ .setDBUrl(dbUrl)
+ .setQuery("select * from " + targetTable)
+ .finish();
+ jdbcInputFormat.open(null);
+
+ int recordCount = 0;
+ while (!jdbcInputFormat.reachedEnd()) {
+ jdbcInputFormat.nextRecord(tuple);
+ Assert.assertEquals("Field 0 should be int", Integer.class, tuple.getField(0).getClass());
+ Assert.assertEquals("Field 1 should be String", String.class, tuple.getField(1).getClass());
+ Assert.assertEquals("Field 2 should be String", String.class, tuple.getField(2).getClass());
+ Assert.assertEquals("Field 3 should be float", Double.class, tuple.getField(3).getClass());
+ Assert.assertEquals("Field 4 should be int", Integer.class, tuple.getField(4).getClass());
+
+ for (int x = 0; x < 5; x++) {
+ Assert.assertEquals(dbData[recordCount][x], tuple.getField(x));
+ }
+
+ recordCount++;
+ }
+ Assert.assertEquals(5, recordCount);
+
+ jdbcInputFormat.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/DevNullLogStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/DevNullLogStream.java b/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/DevNullLogStream.java
new file mode 100644
index 0000000..172f585
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/DevNullLogStream.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.java.record.io.jdbc;
+
+import java.io.OutputStream;
+
+public class DevNullLogStream {
+
+ public static final OutputStream DEV_NULL = new OutputStream() {
+ public void write(int b) {}
+ };
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java b/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java
new file mode 100644
index 0000000..8e0a2c5
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.java.record.io.jdbc;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.junit.Assert;
+
+import org.apache.flink.api.java.record.io.jdbc.JDBCInputFormat;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.types.DoubleValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.StringValue;
+import org.apache.flink.types.Value;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class JDBCInputFormatTest {
+ JDBCInputFormat jdbcInputFormat;
+ Configuration config;
+ static Connection conn;
+ static final Value[][] dbData = {
+ {new IntValue(1001), new StringValue("Java for dummies"), new StringValue("Tan Ah Teck"), new DoubleValue(11.11), new IntValue(11)},
+ {new IntValue(1002), new StringValue("More Java for dummies"), new StringValue("Tan Ah Teck"), new DoubleValue(22.22), new IntValue(22)},
+ {new IntValue(1003), new StringValue("More Java for more dummies"), new StringValue("Mohammad Ali"), new DoubleValue(33.33), new IntValue(33)},
+ {new IntValue(1004), new StringValue("A Cup of Java"), new StringValue("Kumar"), new DoubleValue(44.44), new IntValue(44)},
+ {new IntValue(1005), new StringValue("A Teaspoon of Java"), new StringValue("Kevin Jones"), new DoubleValue(55.55), new IntValue(55)}};
+
+ @BeforeClass
+ public static void setUpClass() {
+ try {
+ prepareDerbyDatabase();
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ private static void prepareDerbyDatabase() throws ClassNotFoundException {
+ System.setProperty("derby.stream.error.field","org.apache.flink.api.java.record.io.jdbc.DevNullLogStream.DEV_NULL");
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ createConnection(dbURL);
+ }
+
+ private static void cleanUpDerbyDatabases() {
+ try {
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ conn = DriverManager.getConnection(dbURL);
+ Statement stat = conn.createStatement();
+ stat.executeUpdate("DROP TABLE books");
+ stat.close();
+ conn.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ /*
+ Loads JDBC derby driver ; creates(if necessary) and populates database.
+ */
+ private static void createConnection(String dbURL) {
+ try {
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ conn = DriverManager.getConnection(dbURL);
+ createTable();
+ insertDataToSQLTables();
+ conn.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ private static void createTable() throws SQLException {
+ StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE books (");
+ sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
+ sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
+ sqlQueryBuilder.append("qty INT DEFAULT NULL,");
+ sqlQueryBuilder.append("PRIMARY KEY (id))");
+
+ Statement stat = conn.createStatement();
+ stat.executeUpdate(sqlQueryBuilder.toString());
+ stat.close();
+
+ sqlQueryBuilder = new StringBuilder("CREATE TABLE bookscontent (");
+ sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
+ sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("content BLOB(10K) DEFAULT NULL,");
+ sqlQueryBuilder.append("PRIMARY KEY (id))");
+
+ stat = conn.createStatement();
+ stat.executeUpdate(sqlQueryBuilder.toString());
+ stat.close();
+ }
+
+ private static void insertDataToSQLTables() throws SQLException {
+ StringBuilder sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
+ sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
+ sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
+ sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
+ sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
+ sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
+
+ Statement stat = conn.createStatement();
+ stat.execute(sqlQueryBuilder.toString());
+ stat.close();
+
+ sqlQueryBuilder = new StringBuilder("INSERT INTO bookscontent (id, title, content) VALUES ");
+ sqlQueryBuilder.append("(1001, 'Java for dummies', CAST(X'7f454c4602' AS BLOB)),");
+ sqlQueryBuilder.append("(1002, 'More Java for dummies', CAST(X'7f454c4602' AS BLOB)),");
+ sqlQueryBuilder.append("(1003, 'More Java for more dummies', CAST(X'7f454c4602' AS BLOB)),");
+ sqlQueryBuilder.append("(1004, 'A Cup of Java', CAST(X'7f454c4602' AS BLOB)),");
+ sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', CAST(X'7f454c4602' AS BLOB))");
+
+ stat = conn.createStatement();
+ stat.execute(sqlQueryBuilder.toString());
+ stat.close();
+ }
+
+
+ @After
+ public void tearDown() {
+ jdbcInputFormat = null;
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidConnection() {
+ jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:idontexist", "select * from books;");
+ jdbcInputFormat.configure(null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidQuery() {
+ jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:ebookshop", "abc");
+ jdbcInputFormat.configure(null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidDBType() {
+ jdbcInputFormat = new JDBCInputFormat("idontexist.Driver", "jdbc:derby:memory:ebookshop", "select * from books;");
+ jdbcInputFormat.configure(null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testUnsupportedSQLType() {
+ jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:ebookshop", "select * from bookscontent");
+ jdbcInputFormat.configure(null);
+ jdbcInputFormat.nextRecord(new Record());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNotConfiguredFormatNext() {
+ jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:ebookshop", "select * from books");
+ jdbcInputFormat.nextRecord(new Record());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNotConfiguredFormatEnd() {
+ jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:ebookshop", "select * from books");
+ jdbcInputFormat.reachedEnd();
+ }
+
+ @Test
+ public void testJDBCInputFormat() throws IOException {
+ jdbcInputFormat = new JDBCInputFormat("org.apache.derby.jdbc.EmbeddedDriver", "jdbc:derby:memory:ebookshop", "select * from books");
+ jdbcInputFormat.configure(null);
+ Record record = new Record();
+ int recordCount = 0;
+ while (!jdbcInputFormat.reachedEnd()) {
+ jdbcInputFormat.nextRecord(record);
+ Assert.assertEquals(5, record.getNumFields());
+ Assert.assertEquals("Field 0 should be int", IntValue.class, record.getField(0, IntValue.class).getClass());
+ Assert.assertEquals("Field 1 should be String", StringValue.class, record.getField(1, StringValue.class).getClass());
+ Assert.assertEquals("Field 2 should be String", StringValue.class, record.getField(2, StringValue.class).getClass());
+ Assert.assertEquals("Field 3 should be float", DoubleValue.class, record.getField(3, DoubleValue.class).getClass());
+ Assert.assertEquals("Field 4 should be int", IntValue.class, record.getField(4, IntValue.class).getClass());
+
+ int[] pos = {0, 1, 2, 3, 4};
+ Value[] values = {new IntValue(), new StringValue(), new StringValue(), new DoubleValue(), new IntValue()};
+ Assert.assertTrue(record.equalsFields(pos, dbData[recordCount], values));
+
+ recordCount++;
+ }
+ Assert.assertEquals(5, recordCount);
+
+ cleanUpDerbyDatabases();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java b/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java
new file mode 100644
index 0000000..c824ea1
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.record.io.jdbc;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.junit.Assert;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.types.DoubleValue;
+import org.apache.flink.types.FloatValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.types.StringValue;
+import org.apache.flink.types.Value;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class JDBCOutputFormatTest {
+ private JDBCInputFormat jdbcInputFormat;
+ private JDBCOutputFormat jdbcOutputFormat;
+
+ private static Connection conn;
+
+ static final Value[][] dbData = {
+ {new IntValue(1001), new StringValue("Java for dummies"), new StringValue("Tan Ah Teck"), new DoubleValue(11.11), new IntValue(11)},
+ {new IntValue(1002), new StringValue("More Java for dummies"), new StringValue("Tan Ah Teck"), new DoubleValue(22.22), new IntValue(22)},
+ {new IntValue(1003), new StringValue("More Java for more dummies"), new StringValue("Mohammad Ali"), new DoubleValue(33.33), new IntValue(33)},
+ {new IntValue(1004), new StringValue("A Cup of Java"), new StringValue("Kumar"), new DoubleValue(44.44), new IntValue(44)},
+ {new IntValue(1005), new StringValue("A Teaspoon of Java"), new StringValue("Kevin Jones"), new DoubleValue(55.55), new IntValue(55)}};
+
+ @BeforeClass
+ public static void setUpClass() {
+ try {
+ System.setProperty("derby.stream.error.field", "org.apache.flink.api.java.record.io.jdbc.DevNullLogStream.DEV_NULL");
+ prepareDerbyInputDatabase();
+ prepareDerbyOutputDatabase();
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ private static void cleanUpDerbyDatabases() {
+ try {
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ conn = DriverManager.getConnection(dbURL);
+ Statement stat = conn.createStatement();
+ stat.executeUpdate("DROP TABLE books");
+ stat.executeUpdate("DROP TABLE newbooks");
+ stat.close();
+ conn.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ private static void prepareDerbyInputDatabase() throws ClassNotFoundException {
+ try {
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ conn = DriverManager.getConnection(dbURL);
+ createTableBooks();
+ insertDataToSQLTables();
+ conn.close();
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ Assert.fail();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ private static void prepareDerbyOutputDatabase() throws ClassNotFoundException {
+ try {
+ String dbURL = "jdbc:derby:memory:ebookshop;create=true";
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ conn = DriverManager.getConnection(dbURL);
+ createTableNewBooks();
+ conn.close();
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ Assert.fail();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ private static void createTableBooks() throws SQLException {
+ StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE books (");
+ sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
+ sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
+ sqlQueryBuilder.append("qty INT DEFAULT NULL,");
+ sqlQueryBuilder.append("PRIMARY KEY (id))");
+
+ Statement stat = conn.createStatement();
+ stat.executeUpdate(sqlQueryBuilder.toString());
+ stat.close();
+ }
+
+ private static void createTableNewBooks() throws SQLException {
+ StringBuilder sqlQueryBuilder = new StringBuilder("CREATE TABLE newbooks (");
+ sqlQueryBuilder.append("id INT NOT NULL DEFAULT 0,");
+ sqlQueryBuilder.append("title VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("author VARCHAR(50) DEFAULT NULL,");
+ sqlQueryBuilder.append("price FLOAT DEFAULT NULL,");
+ sqlQueryBuilder.append("qty INT DEFAULT NULL,");
+ sqlQueryBuilder.append("PRIMARY KEY (id))");
+
+ Statement stat = conn.createStatement();
+ stat.executeUpdate(sqlQueryBuilder.toString());
+ stat.close();
+ }
+
+ private static void insertDataToSQLTables() throws SQLException {
+ StringBuilder sqlQueryBuilder = new StringBuilder("INSERT INTO books (id, title, author, price, qty) VALUES ");
+ sqlQueryBuilder.append("(1001, 'Java for dummies', 'Tan Ah Teck', 11.11, 11),");
+ sqlQueryBuilder.append("(1002, 'More Java for dummies', 'Tan Ah Teck', 22.22, 22),");
+ sqlQueryBuilder.append("(1003, 'More Java for more dummies', 'Mohammad Ali', 33.33, 33),");
+ sqlQueryBuilder.append("(1004, 'A Cup of Java', 'Kumar', 44.44, 44),");
+ sqlQueryBuilder.append("(1005, 'A Teaspoon of Java', 'Kevin Jones', 55.55, 55)");
+
+ Statement stat = conn.createStatement();
+ stat.execute(sqlQueryBuilder.toString());
+ stat.close();
+ }
+
+
+ @After
+ public void tearDown() {
+ jdbcOutputFormat = null;
+ cleanUpDerbyDatabases();
+ }
+
+ @Test
+ public void testJDBCOutputFormat() throws IOException {
+ String sourceTable = "books";
+ String targetTable = "newbooks";
+ String driverPath = "org.apache.derby.jdbc.EmbeddedDriver";
+ String dbUrl = "jdbc:derby:memory:ebookshop";
+
+ Configuration cfg = new Configuration();
+ cfg.setString("driver", driverPath);
+ cfg.setString("url", dbUrl);
+ cfg.setString("query", "insert into " + targetTable + " (id, title, author, price, qty) values (?,?,?,?,?)");
+ cfg.setInteger("fields", 5);
+ cfg.setClass("type0", IntValue.class);
+ cfg.setClass("type1", StringValue.class);
+ cfg.setClass("type2", StringValue.class);
+ cfg.setClass("type3", FloatValue.class);
+ cfg.setClass("type4", IntValue.class);
+
+ jdbcOutputFormat = new JDBCOutputFormat();
+ jdbcOutputFormat.configure(cfg);
+ jdbcOutputFormat.open(0,1);
+
+ jdbcInputFormat = new JDBCInputFormat(
+ driverPath,
+ dbUrl,
+ "select * from " + sourceTable);
+ jdbcInputFormat.configure(null);
+
+ Record record = new Record();
+ while (!jdbcInputFormat.reachedEnd()) {
+ jdbcInputFormat.nextRecord(record);
+ jdbcOutputFormat.writeRecord(record);
+ }
+
+ jdbcOutputFormat.close();
+ jdbcInputFormat.close();
+
+ jdbcInputFormat = new JDBCInputFormat(
+ driverPath,
+ dbUrl,
+ "select * from " + targetTable);
+ jdbcInputFormat.configure(null);
+
+ int recordCount = 0;
+ while (!jdbcInputFormat.reachedEnd()) {
+ jdbcInputFormat.nextRecord(record);
+ Assert.assertEquals(5, record.getNumFields());
+ Assert.assertEquals("Field 0 should be int", IntValue.class, record.getField(0, IntValue.class).getClass());
+ Assert.assertEquals("Field 1 should be String", StringValue.class, record.getField(1, StringValue.class).getClass());
+ Assert.assertEquals("Field 2 should be String", StringValue.class, record.getField(2, StringValue.class).getClass());
+ Assert.assertEquals("Field 3 should be float", DoubleValue.class, record.getField(3, DoubleValue.class).getClass());
+ Assert.assertEquals("Field 4 should be int", IntValue.class, record.getField(4, IntValue.class).getClass());
+
+ int[] pos = {0, 1, 2, 3, 4};
+ Value[] values = {new IntValue(), new StringValue(), new StringValue(), new DoubleValue(), new IntValue()};
+ Assert.assertTrue(record.equalsFields(pos, dbData[recordCount], values));
+
+ recordCount++;
+ }
+ Assert.assertEquals(5, recordCount);
+
+ jdbcInputFormat.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/test/resources/log4j-test.properties b/flink-staging/flink-jdbc/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..2fb9345
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/test/resources/log4j-test.properties
@@ -0,0 +1,19 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-jdbc/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-jdbc/src/test/resources/logback-test.xml b/flink-staging/flink-jdbc/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..8b3bb27
--- /dev/null
+++ b/flink-staging/flink-jdbc/src/test/resources/logback-test.xml
@@ -0,0 +1,29 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/pom.xml b/flink-staging/flink-spargel/pom.xml
new file mode 100644
index 0000000..a0f0f73
--- /dev/null
+++ b/flink-staging/flink-spargel/pom.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-staging</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-spargel</artifactId>
+ <name>flink-spargel</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessageIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessageIterator.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessageIterator.java
new file mode 100644
index 0000000..535732d
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessageIterator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java;
+
+import java.util.Iterator;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+
+/**
+ * An iterator that returns messages. The iterator is {@link java.lang.Iterable} at the same time to support
+ * the <i>foreach</i> syntax.
+ */
+public final class MessageIterator<Message> implements Iterator<Message>, Iterable<Message>, java.io.Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private transient Iterator<Tuple2<?, Message>> source;
+
+
+ final void setSource(Iterator<Tuple2<?, Message>> source) {
+ this.source = source;
+ }
+
+ @Override
+ public final boolean hasNext() {
+ return this.source.hasNext();
+ }
+
+ @Override
+ public final Message next() {
+ return this.source.next().f1;
+ }
+
+ @Override
+ public final void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterator<Message> iterator() {
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessagingFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessagingFunction.java b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessagingFunction.java
new file mode 100644
index 0000000..41c2720
--- /dev/null
+++ b/flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/MessagingFunction.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.spargel.java;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Iterator;
+
+import org.apache.flink.api.common.aggregators.Aggregator;
+import org.apache.flink.api.common.functions.IterationRuntimeContext;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.spargel.java.OutgoingEdge;
+import org.apache.flink.types.Value;
+import org.apache.flink.util.Collector;
+
+/**
+ * The base class for functions that produce messages between vertices as a part of a {@link VertexCentricIteration}.
+ *
+ * @param <VertexKey> The type of the vertex key (the vertex identifier).
+ * @param <VertexValue> The type of the vertex value (the state of the vertex).
+ * @param <Message> The type of the message sent between vertices along the edges.
+ * @param <EdgeValue> The type of the values that are associated with the edges.
+ */
+public abstract class MessagingFunction<VertexKey extends Comparable<VertexKey>, VertexValue, Message, EdgeValue> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ // --------------------------------------------------------------------------------------------
+ // Public API Methods
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * This method is invoked once per superstep for each vertex that was changed in that superstep.
+ * It needs to produce the messages that will be received by vertices in the next superstep.
+ *
+ * @param vertexKey The key of the vertex that was changed.
+ * @param vertexValue The value (state) of the vertex that was changed.
+ *
+ * @throws Exception The computation may throw exceptions, which causes the superstep to fail.
+ */
+ public abstract void sendMessages(VertexKey vertexKey, VertexValue vertexValue) throws Exception;
+
+ /**
+ * This method is executed one per superstep before the vertex update function is invoked for each vertex.
+ *
+ * @throws Exception Exceptions in the pre-superstep phase cause the superstep to fail.
+ */
+ public void preSuperstep() throws Exception {}
+
+ /**
+ * This method is executed one per superstep after the vertex update function has been invoked for each vertex.
+ *
+ * @throws Exception Exceptions in the post-superstep phase cause the superstep to fail.
+ */
+ public void postSuperstep() throws Exception {}
+
+
+ /**
+ * Gets an {@link java.lang.Iterable} with all outgoing edges. This method is mutually exclusive with
+ * {@link #sendMessageToAllNeighbors(Object)} and may be called only once.
+ *
+ * @return An iterator with all outgoing edges.
+ */
+ @SuppressWarnings("unchecked")
+ public Iterable<OutgoingEdge<VertexKey, EdgeValue>> getOutgoingEdges() {
+ if (edgesUsed) {
+ throw new IllegalStateException("Can use either 'getOutgoingEdges()' or 'sendMessageToAllTargets()' exactly once.");
+ }
+ edgesUsed = true;
+
+ if (this.edgeWithValueIter != null) {
+ this.edgeWithValueIter.set((Iterator<Tuple3<VertexKey, VertexKey, EdgeValue>>) edges);
+ return this.edgeWithValueIter;
+ } else {
+ this.edgeNoValueIter.set((Iterator<Tuple2<VertexKey, VertexKey>>) edges);
+ return this.edgeNoValueIter;
+ }
+ }
+
+ /**
+ * Sends the given message to all vertices that are targets of an outgoing edge of the changed vertex.
+ * This method is mutually exclusive to the method {@link #getOutgoingEdges()} and may be called only once.
+ *
+ * @param m The message to send.
+ */
+ public void sendMessageToAllNeighbors(Message m) {
+ if (edgesUsed) {
+ throw new IllegalStateException("Can use either 'getOutgoingEdges()' or 'sendMessageToAllTargets()' exactly once.");
+ }
+
+ edgesUsed = true;
+
+ outValue.f1 = m;
+
+ while (edges.hasNext()) {
+ Tuple next = (Tuple) edges.next();
+ VertexKey k = next.getField(1);
+ outValue.f0 = k;
+ out.collect(outValue);
+ }
+ }
+
+ /**
+ * Sends the given message to the vertex identified by the given key. If the target vertex does not exist,
+ * the next superstep will cause an exception due to a non-deliverable message.
+ *
+ * @param target The key (id) of the target vertex to message.
+ * @param m The message.
+ */
+ public void sendMessageTo(VertexKey target, Message m) {
+ outValue.f0 = target;
+ outValue.f1 = m;
+ out.collect(outValue);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the number of the superstep, starting at <tt>1</tt>.
+ *
+ * @return The number of the current superstep.
+ */
+ public int getSuperstepNumber() {
+ return this.runtimeContext.getSuperstepNumber();
+ }
+
+ /**
+ * Gets the iteration aggregator registered under the given name. The iteration aggregator is combines
+ * all aggregates globally once per superstep and makes them available in the next superstep.
+ *
+ * @param name The name of the aggregator.
+ * @return The aggregator registered under this name, or null, if no aggregator was registered.
+ */
+ public <T extends Aggregator<?>> T getIterationAggregator(String name) {
+ return this.runtimeContext.<T>getIterationAggregator(name);
+ }
+
+ /**
+ * Get the aggregated value that an aggregator computed in the previous iteration.
+ *
+ * @param name The name of the aggregator.
+ * @return The aggregated value of the previous iteration.
+ */
+ public <T extends Value> T getPreviousIterationAggregate(String name) {
+ return this.runtimeContext.<T>getPreviousIterationAggregate(name);
+ }
+
+ /**
+ * Gets the broadcast data set registered under the given name. Broadcast data sets
+ * are available on all parallel instances of a function. They can be registered via
+ * {@link VertexCentricIteration#addBroadcastSetForMessagingFunction(String, org.apache.flink.api.java.DataSet)}.
+ *
+ * @param name The name under which the broadcast set is registered.
+ * @return The broadcast data set.
+ */
+ public <T> Collection<T> getBroadcastSet(String name) {
+ return this.runtimeContext.<T>getBroadcastVariable(name);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // internal methods and state
+ // --------------------------------------------------------------------------------------------
+
+ private Tuple2<VertexKey, Message> outValue;
+
+ private IterationRuntimeContext runtimeContext;
+
+ private Iterator<?> edges;
+
+ private Collector<Tuple2<VertexKey, Message>> out;
+
+ private EdgesIteratorNoEdgeValue<VertexKey, EdgeValue> edgeNoValueIter;
+
+ private EdgesIteratorWithEdgeValue<VertexKey, EdgeValue> edgeWithValueIter;
+
+ private boolean edgesUsed;
+
+
+ void init(IterationRuntimeContext context, boolean hasEdgeValue) {
+ this.runtimeContext = context;
+ this.outValue = new Tuple2<VertexKey, Message>();
+
+ if (hasEdgeValue) {
+ this.edgeWithValueIter = new EdgesIteratorWithEdgeValue<VertexKey, EdgeValue>();
+ } else {
+ this.edgeNoValueIter = new EdgesIteratorNoEdgeValue<VertexKey, EdgeValue>();
+ }
+ }
+
+ void set(Iterator<?> edges, Collector<Tuple2<VertexKey, Message>> out) {
+ this.edges = edges;
+ this.out = out;
+ this.edgesUsed = false;
+ }
+
+
+
+ private static final class EdgesIteratorNoEdgeValue<VertexKey extends Comparable<VertexKey>, EdgeValue>
+ implements Iterator<OutgoingEdge<VertexKey, EdgeValue>>, Iterable<OutgoingEdge<VertexKey, EdgeValue>>
+ {
+ private Iterator<Tuple2<VertexKey, VertexKey>> input;
+
+ private OutgoingEdge<VertexKey, EdgeValue> edge = new OutgoingEdge<VertexKey, EdgeValue>();
+
+
+ void set(Iterator<Tuple2<VertexKey, VertexKey>> input) {
+ this.input = input;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return input.hasNext();
+ }
+
+ @Override
+ public OutgoingEdge<VertexKey, EdgeValue> next() {
+ Tuple2<VertexKey, VertexKey> next = input.next();
+ edge.set(next.f1, null);
+ return edge;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterator<OutgoingEdge<VertexKey, EdgeValue>> iterator() {
+ return this;
+ }
+ }
+
+
+ private static final class EdgesIteratorWithEdgeValue<VertexKey extends Comparable<VertexKey>, EdgeValue>
+ implements Iterator<OutgoingEdge<VertexKey, EdgeValue>>, Iterable<OutgoingEdge<VertexKey, EdgeValue>>
+ {
+ private Iterator<Tuple3<VertexKey, VertexKey, EdgeValue>> input;
+
+ private OutgoingEdge<VertexKey, EdgeValue> edge = new OutgoingEdge<VertexKey, EdgeValue>();
+
+ void set(Iterator<Tuple3<VertexKey, VertexKey, EdgeValue>> input) {
+ this.input = input;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return input.hasNext();
+ }
+
+ @Override
+ public OutgoingEdge<VertexKey, EdgeValue> next() {
+ Tuple3<VertexKey, VertexKey, EdgeValue> next = input.next();
+ edge.set(next.f1, next.f2);
+ return edge;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ @Override
+ public Iterator<OutgoingEdge<VertexKey, EdgeValue>> iterator() {
+ return this;
+ }
+ }
+}
[34/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java
deleted file mode 100644
index c224ad4..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import java.io.Serializable;
-
-/**
- * An eviction policy specifies under which condition data points should be
- * deleted from the buffer. Deletions must be done only in the order the
- * elements arrived. Therefore, the policy only returns the number of elements
- * to evict on each element arrival.
- *
- * @param <DATA>
- * the type of the data handled by this policy
- */
-public interface EvictionPolicy<DATA> extends Serializable {
-
- /**
- * Proves if and how many elements should be deleted from the element
- * buffer. The eviction takes place after the trigger and after the call to
- * the UDF but before the adding of the new data point.
- *
- * @param datapoint
- * data point the data point which arrived
- * @param triggered
- * Information whether the UDF was triggered or not
- * @param bufferSize
- * The current size of the element buffer at the invokable
- * @return The number of elements to be deleted from the buffer
- */
- public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
deleted file mode 100644
index ee8a3cb..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
-
-/**
- * This policy can be used to trigger and evict based on a punctuation which is
- * present within the arriving data. Using this policy, one can react on an
- * externally defined arbitrary windowing semantic.
- *
- * In case this policy is used for eviction, the complete buffer will get
- * deleted in case the punctuation is detected.
- *
- * By default this policy does not react on fake elements. Wrap it in an
- * {@link ActiveEvictionPolicyWrapper} to make it react on punctuation even in
- * fake elements.
- *
- * @param <IN>
- * The type of the input data handled by this policy. An
- * {@link Extractor} can be used to extract DATA for IN.
- * @param <DATA>
- * The type of the punctuation. An {@link Extractor} can be used to
- * extract DATA for IN.
- */
-public class PunctuationPolicy<IN, DATA> implements CloneableTriggerPolicy<IN>,
- CloneableEvictionPolicy<IN> {
-
- /**
- * auto generated version id
- */
- private static final long serialVersionUID = -8845130188912602498L;
- private int counter = 0;
- private Extractor<IN, DATA> extractor;
- private DATA punctuation;
-
- /**
- * Creates the punctuation policy without using any extractor. To make this
- * work IN and DATA must not be different types.
- *
- * @param punctuation
- * the punctuation which leads to trigger/evict.
- */
- public PunctuationPolicy(DATA punctuation) {
- this(punctuation, null);
- }
-
- /**
- * Creates the punctuation policy which uses the specified extractor to
- * isolate the punctuation from the data.
- *
- * @param punctuation
- * the punctuation which leads to trigger/evict.
- * @param extractor
- * An {@link Extractor} which converts IN to DATA.
- */
- public PunctuationPolicy(DATA punctuation, Extractor<IN, DATA> extractor) {
- this.punctuation = punctuation;
- this.extractor = extractor;
- }
-
- @Override
- public int notifyEviction(IN datapoint, boolean triggered, int bufferSize) {
- if (notifyTrigger(datapoint)) {
- int tmp = counter;
- // As the current will be add after the eviction the counter needs
- // to be set to one already
- counter = 1;
- return tmp;
- } else {
- counter++;
- return 0;
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public boolean notifyTrigger(IN datapoint) {
- DATA tmp;
-
- // eventually extract data
- if (extractor == null) {
- // unchecked convert (cannot check it here)
- tmp = (DATA) datapoint;
- } else {
- tmp = extractor.extract(datapoint);
- }
-
- // compare data with punctuation
- if (punctuation.equals(tmp)) {
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public PunctuationPolicy<IN, DATA> clone() {
- return new PunctuationPolicy<IN, DATA>(punctuation, extractor);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
deleted file mode 100644
index 16c30fc..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import java.util.LinkedList;
-
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-
-/**
- * This eviction policy evicts all elements which are older then a specified
- * time. The time is measured using a given {@link Timestamp} implementation. A
- * point in time is always represented as long. Therefore, the granularity can
- * be set as long value as well.
- *
- * @param <DATA>
- * The type of the incoming data points which are processed by this
- * policy.
- */
-public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
- CloneableEvictionPolicy<DATA> {
-
- /**
- * auto generated version id
- */
- private static final long serialVersionUID = -1457476766124518220L;
-
- private long granularity;
- private TimestampWrapper<DATA> timestampWrapper;
- private LinkedList<Long> buffer = new LinkedList<Long>();
-
- /**
- * This eviction policy evicts all elements which are older than a specified
- * time. The time is measured using a given {@link Timestamp}
- * implementation. A point in time is always represented as long. Therefore,
- * the granularity can be set as long value as well. If this value is set to
- * 2 the policy will evict all elements which are older as 2.
- *
- * <code>
- * while (time(firstInBuffer)<current time-granularity){
- * evict firstInBuffer;
- * }
- * </code>
- *
- * @param granularity
- * The granularity of the eviction. If this value is set to 2 the
- * policy will evict all elements which are older as 2(if
- * (time(X)<current time-granularity) evict X).
- * @param timestampWrapper
- * The {@link TimestampWrapper} to measure the time with. This
- * can be either user defined of provided by the API.
- */
- public TimeEvictionPolicy(long granularity, TimestampWrapper<DATA> timestampWrapper) {
- this.timestampWrapper = timestampWrapper;
- this.granularity = granularity;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public int notifyEvictionWithFakeElement(Object datapoint, int bufferSize) {
- checkForDeleted(bufferSize);
-
- long threshold;
- try {
- threshold = (Long) datapoint - granularity;
- } catch (ClassCastException e) {
- threshold = timestampWrapper.getTimestamp((DATA) datapoint) - granularity;
- }
-
- // return result
- return deleteAndCountExpired(threshold);
-
- }
-
- @Override
- public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
-
- checkForDeleted(bufferSize);
-
- // remember timestamp
- long time = timestampWrapper.getTimestamp(datapoint);
-
- // delete and count expired tuples
- long threshold = time - granularity;
- int counter = deleteAndCountExpired(threshold);
-
- // Add current element to buffer
- buffer.add(time);
-
- // return result
- return counter;
-
- }
-
- private void checkForDeleted(int bufferSize) {
- // check for deleted tuples (deletes by other policies)
- while (bufferSize < this.buffer.size()) {
- this.buffer.removeFirst();
- }
- }
-
- private int deleteAndCountExpired(long threshold) {
- int counter = 0;
- while (!buffer.isEmpty()) {
-
- if (buffer.getFirst() <= threshold) {
- buffer.removeFirst();
- counter++;
- } else {
- break;
- }
- }
- return counter;
-
- }
-
- @Override
- public TimeEvictionPolicy<DATA> clone() {
- return new TimeEvictionPolicy<DATA>(granularity, timestampWrapper);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
deleted file mode 100644
index 1e91b8e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import java.util.LinkedList;
-
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-
-/**
- * This trigger policy triggers with regard to the time. The is measured using a
- * given {@link Timestamp} implementation. A point in time is always represented
- * as long. Therefore, parameters such as granularity and delay can be set as
- * long value as well.
- *
- * @param <DATA>
- * The type of the incoming data points which are processed by this
- * policy.
- */
-public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
- CloneableTriggerPolicy<DATA> {
-
- /**
- * auto generated version id
- */
- private static final long serialVersionUID = -5122753802440196719L;
-
- protected long startTime;
- protected long granularity;
- protected TimestampWrapper<DATA> timestampWrapper;
- protected long delay;
-
- /**
- * This trigger policy triggers with regard to the time. The is measured
- * using a given {@link Timestamp} implementation. A point in time is always
- * represented as long. Therefore, parameters such as granularity can be set
- * as long value as well. If this value for the granularity is set to 2 for
- * example, the policy will trigger at every second point in time.
- *
- * @param granularity
- * The granularity of the trigger. If this value is set to x the
- * policy will trigger at every x-th time point
- * @param timestampWrapper
- * The {@link TimestampWrapper} to measure the time with. This
- * can be either user defined of provided by the API.
- * @param timeWrapper
- * This policy creates fake elements to not miss windows in case
- * no element arrived within the duration of the window. This
- * extractor should wrap a long into such an element of type
- * DATA.
- */
- public TimeTriggerPolicy(long granularity, TimestampWrapper<DATA> timestampWrapper) {
- this(granularity, timestampWrapper, 0);
- }
-
- /**
- * This is mostly the same as
- * {@link TimeTriggerPolicy#TimeTriggerPolicy(long, Timestamp)}. In addition
- * to granularity and timestamp a delay can be specified for the first
- * trigger. If the start time given by the timestamp is x, the delay is y,
- * and the granularity is z, the first trigger will happen at x+y+z.
- *
- * @param granularity
- * The granularity of the trigger. If this value is set to 2 the
- * policy will trigger at every second time point
- * @param timestampWrapper
- * The {@link TimestampWrapper} to measure the time with. This
- * can be either user defined of provided by the API.
- * @param delay
- * A delay for the first trigger. If the start time given by the
- * timestamp is x, the delay is y, and the granularity is z, the
- * first trigger will happen at x+y+z.
- * @param timeWrapper
- * This policy creates fake elements to not miss windows in case
- * no element arrived within the duration of the window. This
- * extractor should wrap a long into such an element of type
- * DATA.
- */
- public TimeTriggerPolicy(long granularity, TimestampWrapper<DATA> timestampWrapper, long delay) {
- this.startTime = timestampWrapper.getStartTime() + delay;
- this.timestampWrapper = timestampWrapper;
- this.granularity = granularity;
- this.delay = delay;
- }
-
- /**
- * This method checks if we missed a window end. If this is the case we
- * trigger the missed windows using fake elements.
- */
- @Override
- public synchronized Object[] preNotifyTrigger(DATA datapoint) {
- LinkedList<Object> fakeElements = new LinkedList<Object>();
- // check if there is more then one window border missed
- // use > here. In case >= would fit, the regular call will do the job.
- while (timestampWrapper.getTimestamp(datapoint) >= startTime + granularity) {
- startTime += granularity;
- fakeElements.add(startTime - 1);
- }
- return (Object[]) fakeElements.toArray();
- }
-
- /**
- * In case {@link DefaultTimeStamp} is used, a runnable is returned which
- * triggers based on the current system time. If any other time measure is
- * used the method return null.
- *
- * @param callback
- * The object which is takes the callbacks for adding fake
- * elements out of the runnable.
- * @return A runnable is returned which triggers based on the current system
- * time. If any other time measure is used the method return null.
- */
- @Override
- public Runnable createActiveTriggerRunnable(ActiveTriggerCallback callback) {
- if (this.timestampWrapper.isDefaultTimestamp()) {
- return new TimeCheck(callback);
- } else {
- return null;
- }
- }
-
- /**
- * This method is only called in case the runnable triggers a window end
- * according to the {@link DefaultTimeStamp}.
- *
- * @param callback
- * The callback object.
- */
- private synchronized void activeFakeElementEmission(ActiveTriggerCallback callback) {
-
- // start time is excluded, but end time is included: >=
- if (System.currentTimeMillis() >= startTime + granularity) {
- startTime += granularity;
- callback.sendFakeElement(startTime - 1);
- }
-
- }
-
- private class TimeCheck implements Runnable {
- ActiveTriggerCallback callback;
-
- public TimeCheck(ActiveTriggerCallback callback) {
- this.callback = callback;
- }
-
- @Override
- public void run() {
- while (true) {
- // wait for the specified granularity
- try {
- Thread.sleep(granularity);
- } catch (InterruptedException e) {
- // ignore it...
- }
- // Trigger using the respective methods. Methods are
- // synchronized to prevent race conditions between real and fake
- // elements at the policy.
- activeFakeElementEmission(callback);
- }
- }
- }
-
- @Override
- public synchronized boolean notifyTrigger(DATA datapoint) {
- long recordTime = timestampWrapper.getTimestamp(datapoint);
- if (recordTime >= startTime + granularity) {
- if (granularity != 0) {
- startTime = recordTime - ((recordTime - startTime) % granularity);
- }
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public TimeTriggerPolicy<DATA> clone() {
- return new TimeTriggerPolicy<DATA>(granularity, timestampWrapper, delay);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java
deleted file mode 100644
index c212df6..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import java.io.Serializable;
-
-/**
- * Proves and returns if a new window should be started. In case the trigger
- * occurs (return value true) the UDF will be executed on the current element
- * buffer without the last added element which is provided as parameter. This
- * element will be added to the buffer after the execution of the UDF.
- *
- * @param <DATA>
- * The data type which can be handled by this policy
- */
-public interface TriggerPolicy<DATA> extends Serializable {
-
- /**
- * Proves and returns if a new window should be started. In case the trigger
- * occurs (return value true) the UDF will be executed on the current
- * element buffer without the last added element which is provided as
- * parameter. This element will be added to the buffer after the execution
- * of the UDF.
- *
- * There are possibly different strategies for eviction and triggering: 1)
- * including last data point: Better/faster for count eviction 2) excluding
- * last data point: Essentially required for time based eviction and delta
- * rules As 2) is required for some policies and the benefit of using 1) is
- * small for the others, policies are implemented according to 2).
- *
- * @param datapoint
- * the data point which arrived
- * @return true if the current windows should be closed, otherwise false. In
- * true case the given data point will be part of the next window
- * and will not be included in the current one.
- */
- public boolean notifyTrigger(DATA datapoint);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
deleted file mode 100644
index 9a2e3c9..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-/**
- * This eviction policy deletes all elements from the buffer in case a trigger
- * occurred. Therefore, it is the default eviction policy to be used for any
- * tumbling window.
- *
- * By default this policy does not react on fake elements. Wrap it in an
- * {@link ActiveEvictionPolicyWrapper} to make it clearing the buffer even on
- * fake elements.
- *
- * @param <DATA>
- * The type of the data points which is handled by this policy
- */
-public class TumblingEvictionPolicy<DATA> implements CloneableEvictionPolicy<DATA> {
-
- /**
- * Auto generated version ID
- */
- private static final long serialVersionUID = -4018019069267281155L;
-
- /**
- * Counter for the current number of elements in the buffer
- */
- private int counter = 0;
-
- /**
- * This is the default constructor providing no special functionality. This
- * eviction policy deletes all elements from the buffer in case a trigger
- * occurred. Therefore, it is the default eviction policy to be used for any
- * tumbling window.
- */
- public TumblingEvictionPolicy() {
- // default constructor, no further logic needed
- }
-
- /**
- * This constructor allows to set a custom start value for the element
- * counter.
- *
- * This eviction policy deletes all elements from the buffer in case a
- * trigger occurred. Therefore, it is the default eviction policy to be used
- * for any tumbling window.
- *
- * @param startValue
- * A start value for the element counter
- */
- public TumblingEvictionPolicy(int startValue) {
- this.counter = startValue;
- }
-
- /**
- * Deletes all elements from the buffer in case the trigger occurred.
- */
- @Override
- public int notifyEviction(Object datapoint, boolean triggered, int bufferSize) {
- if (triggered) {
- // The current data point will be part of the next window!
- // Therefore the counter needs to be set to one already.
- int tmpCounter = counter;
- counter = 1;
- return tmpCounter;
- } else {
- counter++;
- return 0;
- }
- }
-
- @Override
- public TumblingEvictionPolicy<DATA> clone() {
- return new TumblingEvictionPolicy<DATA>(counter);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/BlockingQueueBroker.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/BlockingQueueBroker.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/BlockingQueueBroker.java
deleted file mode 100755
index 3ee2508..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/BlockingQueueBroker.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.io;
-
-import java.util.concurrent.BlockingQueue;
-
-import org.apache.flink.runtime.iterative.concurrent.Broker;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-@SuppressWarnings("rawtypes")
-public class BlockingQueueBroker extends Broker<BlockingQueue<StreamRecord>> {
- /**
- * Singleton instance
- */
- private static final BlockingQueueBroker INSTANCE = new BlockingQueueBroker();
-
- private BlockingQueueBroker() {
- }
-
- /**
- * retrieve singleton instance
- */
- public static Broker<BlockingQueue<StreamRecord>> instance() {
- return INSTANCE;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoReaderIterator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoReaderIterator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoReaderIterator.java
deleted file mode 100755
index ed90c03..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoReaderIterator.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.io;
-
-import java.io.IOException;
-
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.runtime.plugable.DeserializationDelegate;
-import org.apache.flink.runtime.plugable.ReusingDeserializationDelegate;
-
-/**
- * A CoReaderIterator wraps a {@link CoRecordReader} producing records of two
- * input types.
- */
-public class CoReaderIterator<T1, T2> {
-
- private final CoRecordReader<DeserializationDelegate<T1>, DeserializationDelegate<T2>> reader; // the
- // source
-
- protected final ReusingDeserializationDelegate<T1> delegate1;
- protected final ReusingDeserializationDelegate<T2> delegate2;
-
- public CoReaderIterator(
- CoRecordReader<DeserializationDelegate<T1>, DeserializationDelegate<T2>> reader,
- TypeSerializer<T1> serializer1, TypeSerializer<T2> serializer2) {
- this.reader = reader;
- this.delegate1 = new ReusingDeserializationDelegate<T1>(serializer1);
- this.delegate2 = new ReusingDeserializationDelegate<T2>(serializer2);
- }
-
- public int next(T1 target1, T2 target2) throws IOException {
- this.delegate1.setInstance(target1);
- this.delegate2.setInstance(target2);
-
- try {
- return this.reader.getNextRecord(this.delegate1, this.delegate2);
-
- } catch (InterruptedException e) {
- throw new IOException("Reader interrupted.", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoRecordReader.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoRecordReader.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoRecordReader.java
deleted file mode 100755
index 0b1b373..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/CoRecordReader.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.io;
-
-import org.apache.flink.core.io.IOReadableWritable;
-import org.apache.flink.runtime.event.task.TaskEvent;
-import org.apache.flink.runtime.io.network.api.reader.BufferReaderBase;
-import org.apache.flink.runtime.io.network.api.reader.MutableRecordReader;
-import org.apache.flink.runtime.io.network.api.reader.ReaderBase;
-import org.apache.flink.runtime.io.network.api.serialization.AdaptiveSpanningRecordDeserializer;
-import org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer;
-import org.apache.flink.runtime.io.network.buffer.Buffer;
-import org.apache.flink.runtime.util.event.EventListener;
-
-import java.io.IOException;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * A CoRecordReader wraps {@link MutableRecordReader}s of two different input
- * types to read records effectively.
- */
-@SuppressWarnings("rawtypes")
-public class CoRecordReader<T1 extends IOReadableWritable, T2 extends IOReadableWritable> implements ReaderBase, EventListener<BufferReaderBase> {
-
- private final BufferReaderBase bufferReader1;
-
- private final BufferReaderBase bufferReader2;
-
- private final BlockingQueue<Integer> availableRecordReaders = new LinkedBlockingQueue<Integer>();
-
- private AdaptiveSpanningRecordDeserializer[] reader1RecordDeserializers;
-
- private RecordDeserializer<T1> reader1currentRecordDeserializer;
-
- private AdaptiveSpanningRecordDeserializer[] reader2RecordDeserializers;
-
- private RecordDeserializer<T2> reader2currentRecordDeserializer;
-
- // 0 => none, 1 => reader (T1), 2 => reader (T2)
- private int currentReaderIndex;
-
- private boolean hasRequestedPartitions;
-
- public CoRecordReader(BufferReaderBase bufferReader1, BufferReaderBase bufferReader2) {
- this.bufferReader1 = bufferReader1;
- this.bufferReader2 = bufferReader2;
-
- this.reader1RecordDeserializers = new AdaptiveSpanningRecordDeserializer[bufferReader1.getNumberOfInputChannels()];
- this.reader2RecordDeserializers = new AdaptiveSpanningRecordDeserializer[bufferReader2.getNumberOfInputChannels()];
-
- for (int i = 0; i < reader1RecordDeserializers.length; i++) {
- reader1RecordDeserializers[i] = new AdaptiveSpanningRecordDeserializer<T1>();
- }
-
- for (int i = 0; i < reader2RecordDeserializers.length; i++) {
- reader2RecordDeserializers[i] = new AdaptiveSpanningRecordDeserializer<T2>();
- }
-
- bufferReader1.subscribeToReader(this);
- bufferReader2.subscribeToReader(this);
- }
-
- public void requestPartitionsOnce() throws IOException {
- if (!hasRequestedPartitions) {
- bufferReader1.requestPartitionsOnce();
- bufferReader2.requestPartitionsOnce();
-
- hasRequestedPartitions = true;
- }
- }
-
- @SuppressWarnings("unchecked")
- protected int getNextRecord(T1 target1, T2 target2) throws IOException, InterruptedException {
-
- requestPartitionsOnce();
-
- while (true) {
- if (currentReaderIndex == 0) {
- if ((bufferReader1.isFinished() && bufferReader2.isFinished())) {
- return 0;
- }
-
- currentReaderIndex = getNextReaderIndexBlocking();
- }
-
- if (currentReaderIndex == 1) {
- while (true) {
- if (reader1currentRecordDeserializer != null) {
- RecordDeserializer.DeserializationResult result = reader1currentRecordDeserializer.getNextRecord(target1);
-
- if (result.isBufferConsumed()) {
- reader1currentRecordDeserializer.getCurrentBuffer().recycle();
- reader1currentRecordDeserializer = null;
-
- currentReaderIndex = 0;
- }
-
- if (result.isFullRecord()) {
- return 1;
- }
- }
-
- final Buffer nextBuffer = bufferReader1.getNextBufferBlocking();
- final int channelIndex = bufferReader1.getChannelIndexOfLastBuffer();
-
- if (nextBuffer == null) {
- currentReaderIndex = 0;
-
- break;
- }
-
- reader1currentRecordDeserializer = reader1RecordDeserializers[channelIndex];
- reader1currentRecordDeserializer.setNextBuffer(nextBuffer);
- }
- }
- else if (currentReaderIndex == 2) {
- while (true) {
- if (reader2currentRecordDeserializer != null) {
- RecordDeserializer.DeserializationResult result = reader2currentRecordDeserializer.getNextRecord(target2);
-
- if (result.isBufferConsumed()) {
- reader2currentRecordDeserializer.getCurrentBuffer().recycle();
- reader2currentRecordDeserializer = null;
-
- currentReaderIndex = 0;
- }
-
- if (result.isFullRecord()) {
- return 2;
- }
- }
-
- final Buffer nextBuffer = bufferReader2.getNextBufferBlocking();
- final int channelIndex = bufferReader2.getChannelIndexOfLastBuffer();
-
- if (nextBuffer == null) {
- currentReaderIndex = 0;
-
- break;
- }
-
- reader2currentRecordDeserializer = reader2RecordDeserializers[channelIndex];
- reader2currentRecordDeserializer.setNextBuffer(nextBuffer);
- }
- }
- else {
- throw new IllegalStateException("Bug: unexpected current reader index.");
- }
- }
- }
-
- private int getNextReaderIndexBlocking() throws InterruptedException {
- return availableRecordReaders.take();
- }
-
- // ------------------------------------------------------------------------
- // Data availability notifications
- // ------------------------------------------------------------------------
-
- @Override
- public void onEvent(BufferReaderBase bufferReader) {
- if (bufferReader == bufferReader1) {
- availableRecordReaders.add(1);
- }
- else if (bufferReader == bufferReader2) {
- availableRecordReaders.add(2);
- }
- }
-
- // ------------------------------------------------------------------------
-
- @Override
- public boolean isFinished() {
- return bufferReader1.isFinished() && bufferReader2.isFinished();
- }
-
- @Override
- public void subscribeToTaskEvent(EventListener<TaskEvent> eventListener, Class<? extends TaskEvent> eventType) {
- bufferReader1.subscribeToTaskEvent(eventListener, eventType);
- bufferReader2.subscribeToTaskEvent(eventListener, eventType);
- }
-
- @Override
- public void sendTaskEvent(TaskEvent event) throws IOException, InterruptedException {
- bufferReader1.sendTaskEvent(event);
- bufferReader2.sendTaskEvent(event);
- }
-
- @Override
- public void setIterativeReader() {
- bufferReader1.setIterativeReader();
- bufferReader2.setIterativeReader();
- }
-
- @Override
- public void startNextSuperstep() {
- bufferReader1.startNextSuperstep();
- bufferReader2.startNextSuperstep();
- }
-
- @Override
- public boolean hasReachedEndOfSuperstep() {
- return bufferReader1.hasReachedEndOfSuperstep() && bufferReader2.hasReachedEndOfSuperstep();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
deleted file mode 100755
index b7af589..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.io;
-
-import org.apache.flink.core.io.IOReadableWritable;
-import org.apache.flink.runtime.io.network.api.writer.BufferWriter;
-import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
-import org.apache.flink.runtime.io.network.api.writer.RoundRobinChannelSelector;
-
-import java.io.IOException;
-
-public class StreamRecordWriter<T extends IOReadableWritable> extends RecordWriter<T> {
-
- private long timeout;
-
- private OutputFlusher outputFlusher;
-
- public StreamRecordWriter(BufferWriter writer) {
- this(writer, new RoundRobinChannelSelector<T>(), 1000);
- }
-
- public StreamRecordWriter(BufferWriter writer, ChannelSelector<T> channelSelector) {
- this(writer, channelSelector, 1000);
- }
-
- public StreamRecordWriter(BufferWriter writer, ChannelSelector<T> channelSelector, long timeout) {
- super(writer, channelSelector);
-
- this.timeout = timeout;
- this.outputFlusher = new OutputFlusher();
-
- outputFlusher.start();
- }
-
- public void close() {
- try {
- if (outputFlusher != null) {
- outputFlusher.terminate();
- outputFlusher.join();
- }
-
- flush();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
-
- private class OutputFlusher extends Thread {
- private boolean running = true;
- public void terminate() {
- running = false;
- }
- @Override
- public void run() {
- while (running && !writer.isFinished()) {
- try {
- flush();
- Thread.sleep(timeout);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java
deleted file mode 100644
index 3813c8a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-/**
- * Partitioner that selects all the output channels.
- *
- * @param <T>
- * Type of the Tuple
- */
-public class BroadcastPartitioner<T> extends StreamPartitioner<T> {
- private static final long serialVersionUID = 1L;
-
- int[] returnArray;
- boolean set;
- int setNumber;
-
- public BroadcastPartitioner() {
- super(PartitioningStrategy.BROADCAST);
- }
-
- @Override
- public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
- int numberOfOutputChannels) {
- if (set && setNumber == numberOfOutputChannels) {
- return returnArray;
- } else {
- this.returnArray = new int[numberOfOutputChannels];
- for (int i = 0; i < numberOfOutputChannels; i++) {
- returnArray[i] = i;
- }
- set = true;
- setNumber = numberOfOutputChannels;
- return returnArray;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java
deleted file mode 100644
index 7299397..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-/**
- * Partitioner that distributes the data equally by cycling through the output
- * channels.
- *
- * @param <T>
- * Type of the Tuple
- */
-public class DistributePartitioner<T> extends StreamPartitioner<T> {
- private static final long serialVersionUID = 1L;
-
- private int[] returnArray = new int[] {-1};
-
- public DistributePartitioner(boolean forward) {
- super(forward ? PartitioningStrategy.FORWARD : PartitioningStrategy.DISTRIBUTE);
- }
-
- @Override
- public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
- int numberOfOutputChannels) {
- this.returnArray[0] = (this.returnArray[0] + 1) % numberOfOutputChannels;
-
- return this.returnArray;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java
deleted file mode 100644
index 6bc036e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-/**
- * Partitioner that selects the same (one) channel for two Tuples having a
- * specified fields equal.
- *
- * @param <T>
- * Type of the Tuple
- */
-public class FieldsPartitioner<T> extends StreamPartitioner<T> {
- private static final long serialVersionUID = 1L;
-
- private int[] returnArray = new int[1];;
- KeySelector<T, ?> keySelector;
-
- public FieldsPartitioner(KeySelector<T, ?> keySelector) {
- super(PartitioningStrategy.GROUPBY);
- this.keySelector = keySelector;
- }
-
- @Override
- public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
- int numberOfOutputChannels) {
- returnArray[0] = Math.abs(record.getInstance().getKey(keySelector).hashCode()
- % numberOfOutputChannels);
-
- return returnArray;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java
deleted file mode 100644
index c73ca71..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-//Group to the partitioner with the lowest id
-public class GlobalPartitioner<T> extends StreamPartitioner<T> {
- private static final long serialVersionUID = 1L;
-
- private int[] returnArray = new int[] { 0 };
-
- public GlobalPartitioner() {
- super(PartitioningStrategy.GLOBAL);
- }
-
- @Override
- public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
- int numberOfOutputChannels) {
- return returnArray;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java
deleted file mode 100644
index 318de3f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import java.util.Random;
-
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-/**
- * Partitioner that distributes the data equally by selecting one output channel
- * randomly.
- *
- * @param <T>
- * Type of the Tuple
- */
-public class ShufflePartitioner<T> extends StreamPartitioner<T> {
- private static final long serialVersionUID = 1L;
-
- private Random random = new Random();
-
- private int[] returnArray = new int[1];
-
- public ShufflePartitioner() {
- super(PartitioningStrategy.SHUFFLE);
- }
-
- @Override
- public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
- int numberOfOutputChannels) {
- returnArray[0] = random.nextInt(numberOfOutputChannels);
- return returnArray;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java
deleted file mode 100644
index 19a8dba..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.partitioner;
-
-import java.io.Serializable;
-
-import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-public abstract class StreamPartitioner<T> implements
- ChannelSelector<SerializationDelegate<StreamRecord<T>>>, Serializable {
-
- public enum PartitioningStrategy {
-
- FORWARD, DISTRIBUTE, SHUFFLE, BROADCAST, GLOBAL, GROUPBY;
-
- }
-
- private static final long serialVersionUID = 1L;
- private PartitioningStrategy strategy;
-
- public StreamPartitioner(PartitioningStrategy strategy) {
- this.strategy = strategy;
- }
-
- public PartitioningStrategy getStrategy() {
- return strategy;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/CircularFifoList.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/CircularFifoList.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/CircularFifoList.java
deleted file mode 100644
index 02a628b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/CircularFifoList.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state;
-
-import java.io.Serializable;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-
-/**
- * A simple class, that manages a circular queue with sliding interval. If the
- * queue if full and a new element is added, the elements that belong to the
- * first sliding interval are removed.
- */
-public class CircularFifoList<T> implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private Queue<T> queue;
- private Queue<Long> slideSizes;
- private long counter;
- private Iterable<T> iterable;
-
- public CircularFifoList() {
- this.queue = new LinkedList<T>();
- this.slideSizes = new LinkedList<Long>();
- this.counter = 0;
- this.iterable = new ListIterable();
- }
-
- public void add(T element) {
- queue.add(element);
- counter++;
- }
-
- public void newSlide() {
- slideSizes.add(counter);
- counter = 0;
- }
-
- public void shiftWindow() {
- shiftWindow(1);
- }
-
- public void shiftWindow(int numberOfSlides) {
-
- if (numberOfSlides <= slideSizes.size()) {
- for (int i = 0; i < numberOfSlides; i++) {
- Long firstSlideSize = slideSizes.remove();
-
- for (int j = 0; j < firstSlideSize; j++) {
- queue.remove();
- }
- }
- } else {
- slideSizes.clear();
- queue.clear();
- counter = 0;
- }
-
- }
-
- @SuppressWarnings("unchecked")
- public List<T> getElements(){
- return (List<T>) queue;
- }
-
- public Iterator<T> getIterator() {
- return queue.iterator();
- }
-
- public Iterable<T> getIterable() {
- return iterable;
- }
-
- private class ListIterable implements Iterable<T>, Serializable {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Iterator<T> iterator() {
- return getIterator();
- }
-
- }
-
- public boolean isEmpty() {
- return queue.isEmpty();
- }
-
- @Override
- public String toString() {
- return queue.toString();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/GraphState.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/GraphState.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/GraphState.java
deleted file mode 100644
index 945c0fd..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/GraphState.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * GraphState represents a special {@link MapState} for storing graph
- * structures.
- *
- */
-public class GraphState extends MapState<Integer, Set<Integer>> {
-
- private static final long serialVersionUID = 1L;
-
- public GraphState() {
- state = new HashMap<Integer, Set<Integer>>();
- }
-
- public void insertDirectedEdge(int sourceNode, int targetNode) {
- if (!containsKey(sourceNode)) {
- state.put(sourceNode, new HashSet<Integer>());
- }
- state.get(sourceNode).add(targetNode);
- updatedItems.add(sourceNode);
- removedItems.remove(sourceNode);
- }
-
- public void insertUndirectedEdge(int sourceNode, int targetNode) {
- if (!state.containsKey(sourceNode)) {
- state.put(sourceNode, new HashSet<Integer>());
- }
- if (!state.containsKey(targetNode)) {
- state.put(targetNode, new HashSet<Integer>());
- }
- state.get(sourceNode).add(targetNode);
- state.get(targetNode).add(sourceNode);
-
- updatedItems.add(sourceNode);
- removedItems.remove(sourceNode);
- updatedItems.add(targetNode);
- removedItems.remove(targetNode);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/MapState.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/MapState.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/MapState.java
deleted file mode 100644
index 85aec52..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/MapState.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.streaming.state.checkpoint.MapCheckpoint;
-import org.apache.flink.streaming.state.checkpoint.StateCheckpoint;
-
-/**
- * A Map that can be used as a partitionable operator state, for both fault
- * tolerance and rebalancing in Flink programs. The default repartitioner for
- * this operator repartitions by the hashcodes of the keys in the map. </br>
- * </br> The MapState also allows for incremental (data efficient) checkpointing
- * of the state. The entries in the map should only be modified by using the
- * dedicated methods: {@link #put(Object, Object)},{@link #remove(Object)},
- * {@link #putAll(Map)} and {@link #clear}. Directly modifying the entryset
- * will cause errors when checkpointing.
- *
- * @param <K>
- * The type of the keys.
- * @param <V>
- * The type of the values.
- */
-public class MapState<K, V> extends PartitionableState<Map<K, V>> implements Map<K, V> {
-
- private static final long serialVersionUID = 1L;
-
- protected Set<K> removedItems;
- protected Set<K> updatedItems;
- protected boolean clear;
-
- public MapState() {
- this.state = new HashMap<K, V>();
- this.updatedItems = new HashSet<K>();
- this.removedItems = new HashSet<K>();
- }
-
- @Override
- public StateCheckpoint<Map<K, V>> checkpoint() {
- this.updatedItems.removeAll(removedItems);
- StateCheckpoint<Map<K, V>> checkpoint = new MapCheckpoint<K, V>(this);
- resetHistory();
- return checkpoint;
- }
-
- @Override
- public OperatorState<Map<K, V>> restore(StateCheckpoint<Map<K, V>> checkpoint) {
- this.state = checkpoint.getCheckpointedState();
- resetHistory();
- return this;
- }
-
- @Override
- public OperatorState<Map<K, V>>[] repartition(int numberOfPartitions) {
- @SuppressWarnings("unchecked")
- MapState<K, V>[] states = new MapState[numberOfPartitions];
- for (int i = 0; i < numberOfPartitions; i++) {
- states[i] = new MapState<K, V>();
- }
- for (Entry<K, V> entry : this.state.entrySet()) {
- int partition = Math.abs(entry.getKey().hashCode() % numberOfPartitions);
- states[partition].state.put(entry.getKey(), entry.getValue());
- }
- return states;
- }
-
- @Override
- public OperatorState<Map<K, V>> reBuild(OperatorState<Map<K, V>>... parts) {
- clear();
- for (OperatorState<Map<K, V>> operatorState : parts) {
- putAll(operatorState.state);
- }
- return this;
- }
-
- public void resetHistory() {
- clear = false;
- removedItems.clear();
- updatedItems.clear();
- }
-
- @Override
- public int size() {
- return state.size();
- }
-
- @Override
- public boolean isEmpty() {
- return state.isEmpty();
- }
-
- @Override
- public boolean containsKey(Object key) {
- return state.containsKey(key);
- }
-
- @Override
- public boolean containsValue(Object value) {
- return state.containsValue(value);
- }
-
- @Override
- public V get(Object key) {
- return state.get(key);
- }
-
- @Override
- public V put(K key, V value) {
- updatedItems.add(key);
- removedItems.remove(key);
- return state.put(key, value);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public V remove(Object key) {
- V removed = state.remove(key);
- if (removed != null) {
- removedItems.add((K) key);
- updatedItems.remove((K) key);
- }
-
- return removed;
- }
-
- @Override
- public void putAll(Map<? extends K, ? extends V> m) {
- for (K key : m.keySet()) {
- updatedItems.add(key);
- removedItems.remove(key);
- }
- state.putAll(m);
- }
-
- @Override
- public void clear() {
- clear = true;
- updatedItems.clear();
- removedItems.clear();
- state.clear();
- }
-
- @Override
- public Set<K> keySet() {
- return state.keySet();
- }
-
- @Override
- public Collection<V> values() {
- return state.values();
- }
-
- @Override
- public Set<Entry<K, V>> entrySet() {
- return state.entrySet();
- }
-
- @Override
- public Map<K, V> getState() {
- throw new RuntimeException(
- "State object should be accessed using the Map interface provided by the MapState");
- }
-
- @Override
- public OperatorState<Map<K, V>> setState(Map<K, V> state) {
- throw new RuntimeException(
- "State object should be accessed using the Map interface provided by the MapState");
- }
-
- public Set<K> getRemovedItems() {
- return removedItems;
- }
-
- public Set<K> getUpdatedItems() {
- return updatedItems;
- }
-
- public boolean isCleared() {
- return clear;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/NullableCircularBuffer.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/NullableCircularBuffer.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/NullableCircularBuffer.java
deleted file mode 100644
index aa2a1bc..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/NullableCircularBuffer.java
+++ /dev/null
@@ -1,362 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.util.AbstractCollection;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.apache.commons.collections.BoundedCollection;
-import org.apache.commons.collections.Buffer;
-import org.apache.commons.collections.BufferUnderflowException;
-
-@SuppressWarnings("rawtypes")
-public class NullableCircularBuffer extends AbstractCollection implements Buffer,
- BoundedCollection, Serializable {
-
- /** Serialization version */
- private static final long serialVersionUID = 5603722811189451017L;
-
- /** Underlying storage array */
- private transient Object[] elements;
-
- /** Array index of first (oldest) buffer element */
- private transient int start = 0;
-
- /**
- * Index mod maxElements of the array position following the last buffer
- * element. Buffer elements start at elements[start] and "wrap around"
- * elements[maxElements-1], ending at elements[decrement(end)]. For example,
- * elements = {c,a,b}, start=1, end=1 corresponds to the buffer [a,b,c].
- */
- private transient int end = 0;
-
- /** Flag to indicate if the buffer is currently full. */
- private transient boolean full = false;
-
- /** Capacity of the buffer */
- private final int maxElements;
-
- /**
- * Constructs a new <code>BoundedFifoBuffer</code> big enough to hold 32
- * elements.
- */
- public NullableCircularBuffer() {
- this(32);
- }
-
- /**
- * Constructs a new <code>BoundedFifoBuffer</code> big enough to hold the
- * specified number of elements.
- *
- * @param size
- * the maximum number of elements for this fifo
- * @throws IllegalArgumentException
- * if the size is less than 1
- */
- public NullableCircularBuffer(int size) {
- if (size <= 0) {
- throw new IllegalArgumentException("The size must be greater than 0");
- }
- elements = new Object[size];
- maxElements = elements.length;
- }
-
- /**
- * Constructs a new <code>BoundedFifoBuffer</code> big enough to hold all of
- * the elements in the specified collection. That collection's elements will
- * also be added to the buffer.
- *
- * @param coll
- * the collection whose elements to add, may not be null
- * @throws NullPointerException
- * if the collection is null
- */
- @SuppressWarnings("unchecked")
- public NullableCircularBuffer(Collection coll) {
- this(coll.size());
- addAll(coll);
- }
-
- // -----------------------------------------------------------------------
- /**
- * Write the buffer out using a custom routine.
- *
- * @param out
- * the output stream
- * @throws IOException
- */
- private void writeObject(ObjectOutputStream out) throws IOException {
- out.defaultWriteObject();
- out.writeInt(size());
- for (Iterator it = iterator(); it.hasNext();) {
- out.writeObject(it.next());
- }
- }
-
- /**
- * Read the buffer in using a custom routine.
- *
- * @param in
- * the input stream
- * @throws IOException
- * @throws ClassNotFoundException
- */
- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
- in.defaultReadObject();
- elements = new Object[maxElements];
- int size = in.readInt();
- for (int i = 0; i < size; i++) {
- elements[i] = in.readObject();
- }
- start = 0;
- full = (size == maxElements);
- if (full) {
- end = 0;
- } else {
- end = size;
- }
- }
-
- // -----------------------------------------------------------------------
- /**
- * Returns the number of elements stored in the buffer.
- *
- * @return this buffer's size
- */
- public int size() {
- int size = 0;
-
- if (end < start) {
- size = maxElements - start + end;
- } else if (end == start) {
- size = (full ? maxElements : 0);
- } else {
- size = end - start;
- }
-
- return size;
- }
-
- /**
- * Returns true if this buffer is empty; false otherwise.
- *
- * @return true if this buffer is empty
- */
- public boolean isEmpty() {
- return size() == 0;
- }
-
- /**
- * Returns true if this collection is full and no new elements can be added.
- *
- * @return <code>true</code> if the collection is full
- */
- public boolean isFull() {
- return size() == maxElements;
- }
-
- /**
- * Gets the maximum size of the collection (the bound).
- *
- * @return the maximum number of elements the collection can hold
- */
- public int maxSize() {
- return maxElements;
- }
-
- /**
- * Clears this buffer.
- */
- public void clear() {
- full = false;
- start = 0;
- end = 0;
- Arrays.fill(elements, null);
- }
-
- /**
- * Adds the given element to this buffer.
- *
- * @param element
- * the element to add
- * @return true, always
- */
- public boolean add(Object element) {
-
- if (isFull()) {
- remove();
- }
-
- elements[end++] = element;
-
- if (end >= maxElements) {
- end = 0;
- }
-
- if (end == start) {
- full = true;
- }
-
- return true;
- }
-
- /**
- * Returns the least recently inserted element in this buffer.
- *
- * @return the least recently inserted element
- * @throws BufferUnderflowException
- * if the buffer is empty
- */
- public Object get() {
- if (isEmpty()) {
- throw new BufferUnderflowException("The buffer is already empty");
- }
-
- return elements[start];
- }
-
- /**
- * Removes the least recently inserted element from this buffer.
- *
- * @return the least recently inserted element
- * @throws BufferUnderflowException
- * if the buffer is empty
- */
- public Object remove() {
- if (isEmpty()) {
- throw new BufferUnderflowException("The buffer is already empty");
- }
-
- Object element = elements[start];
-
- elements[start++] = null;
-
- if (start >= maxElements) {
- start = 0;
- }
-
- full = false;
-
- return element;
- }
-
- /**
- * Increments the internal index.
- *
- * @param index
- * the index to increment
- * @return the updated index
- */
- private int increment(int index) {
- index++;
- if (index >= maxElements) {
- index = 0;
- }
- return index;
- }
-
- /**
- * Decrements the internal index.
- *
- * @param index
- * the index to decrement
- * @return the updated index
- */
- private int decrement(int index) {
- index--;
- if (index < 0) {
- index = maxElements - 1;
- }
- return index;
- }
-
- /**
- * Returns an iterator over this buffer's elements.
- *
- * @return an iterator over this buffer's elements
- */
- public Iterator iterator() {
- return new Iterator() {
-
- private int index = start;
- private int lastReturnedIndex = -1;
- private boolean isFirst = full;
-
- public boolean hasNext() {
- return isFirst || (index != end);
-
- }
-
- public Object next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- isFirst = false;
- lastReturnedIndex = index;
- index = increment(index);
- return elements[lastReturnedIndex];
- }
-
- public void remove() {
- if (lastReturnedIndex == -1) {
- throw new IllegalStateException();
- }
-
- // First element can be removed quickly
- if (lastReturnedIndex == start) {
- NullableCircularBuffer.this.remove();
- lastReturnedIndex = -1;
- return;
- }
-
- int pos = lastReturnedIndex + 1;
- if (start < lastReturnedIndex && pos < end) {
- // shift in one part
- System.arraycopy(elements, pos, elements, lastReturnedIndex, end - pos);
- } else {
- // Other elements require us to shift the subsequent
- // elements
- while (pos != end) {
- if (pos >= maxElements) {
- elements[pos - 1] = elements[0];
- pos = 0;
- } else {
- elements[decrement(pos)] = elements[pos];
- pos = increment(pos);
- }
- }
- }
-
- lastReturnedIndex = -1;
- end = decrement(end);
- elements[end] = null;
- full = false;
- index = decrement(index);
- }
-
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/OperatorState.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/OperatorState.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/OperatorState.java
deleted file mode 100644
index a0cedba..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/OperatorState.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state;
-
-import java.io.Serializable;
-
-import org.apache.flink.streaming.state.checkpoint.StateCheckpoint;
-
-/**
- * Abstract class for representing operator states in Flink programs. By
- * implementing the methods declared in this abstraction the state of the
- * operator can be checkpointed by the fault tolerance mechanism.
- *
- * @param <T>
- * The type of the operator state.
- */
-public abstract class OperatorState<T> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- protected T state;
-
- /**
- * Constructor used for initializing the state. In case of failure, the
- * state will be reinitialized using this constructor, then
- * {@link #restore(StateCheckpoint)} will be used to restore from the last
- * available backup.
- */
- public OperatorState() {
- state = null;
- }
-
- /**
- * Initializes the state using the given state object.
- *
- * @param initialState
- * The initial state object
- */
- public OperatorState(T initialState) {
- state = initialState;
- }
-
- /**
- * Returns the currently stored state object.
- *
- * @return The state.
- */
- public T getState() {
- return state;
- }
-
- /**
- * Sets the current state object.
- *
- * @param state
- * The new state object.
- * @return The operator state with the new state object set.
- */
- public OperatorState<T> setState(T state) {
- this.state = state;
- return this;
- }
-
- /**
- * Creates a {@link StateCheckpoint} that will be used to backup the state
- * for failure recovery.
- *
- * @return The {@link StateCheckpoint} created.
- */
- public abstract StateCheckpoint<T> checkpoint();
-
- /**
- * Restores the state from the given {@link StateCheckpoint}.
- *
- * @param checkpoint
- * The checkpoint to restore from
- * @return The restored operator.
- */
- public abstract OperatorState<T> restore(StateCheckpoint<T> checkpoint);
-
- @Override
- public String toString() {
- return state.toString();
- }
-
- public boolean stateEquals(OperatorState<T> other) {
- return state.equals(other.state);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/PartitionableState.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/PartitionableState.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/PartitionableState.java
deleted file mode 100644
index c58c545..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/PartitionableState.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state;
-
-/**
- * Base class for representing operator states that can be repartitioned for
- * state state and load balancing.
- *
- * @param <T>
- * The type of the operator state.
- */
-public abstract class PartitionableState<T> extends OperatorState<T> {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * Repartitions(divides) the current state into the given number of new
- * partitions. The created partitions will be used to redistribute then
- * rebuild the state among the parallel instances of the operator. The
- * implementation should reflect the partitioning of the input values to
- * maintain correct operator behavior.
- *
- * </br> </br> It is also assumed that if we would {@link #reBuild} the
- * repartitioned state we would basically get the same as before.
- *
- *
- * @param numberOfPartitions
- * The desired number of partitions. The method must return an
- * array of that size.
- * @return The array containing the state part for each partition.
- */
- public abstract OperatorState<T>[] repartition(int numberOfPartitions);
-
- /**
- * Rebuilds the current state partition from the given parts. Used for
- * building the state after a re-balance phase.
- *
- * @param parts
- * The state parts that will be used to rebuild the current
- * partition.
- * @return The rebuilt operator state.
- */
- public abstract OperatorState<T> reBuild(OperatorState<T>... parts);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/SimpleState.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/SimpleState.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/SimpleState.java
deleted file mode 100644
index 7ae1f81..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/SimpleState.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state;
-
-import org.apache.flink.streaming.state.checkpoint.StateCheckpoint;
-
-/**
- * Basic {@link OperatorState} for storing and updating simple objects. By default the
- * whole stored object is checkpointed at each backup. Override checkpoint to
- * allow a more fine-grained behavior.
- *
- * @param <T>
- * The type of the stored state object.
- */
-public class SimpleState<T> extends OperatorState<T> {
-
- private static final long serialVersionUID = 1L;
-
- public SimpleState() {
- super();
- }
-
- public SimpleState(T initialState) {
- super(initialState);
- }
-
- @Override
- public StateCheckpoint<T> checkpoint() {
- return new StateCheckpoint<T>(this);
- }
-
- @Override
- public OperatorState<T> restore(StateCheckpoint<T> checkpoint) {
- this.state = checkpoint.getCheckpointedState();
- return this;
- }
-
-}
[23/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/utils/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/utils/HadoopUtils.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/utils/HadoopUtils.java
new file mode 100644
index 0000000..2d2f518
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/utils/HadoopUtils.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.utils;
+
+import java.lang.reflect.Constructor;
+import java.util.Map;
+
+import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+import org.apache.hadoop.mapred.TaskAttemptID;
+
+
+public class HadoopUtils {
+
+ /**
+ * Merge HadoopConfiguration into JobConf. This is necessary for the HDFS configuration.
+ */
+ public static void mergeHadoopConf(JobConf jobConf) {
+ org.apache.hadoop.conf.Configuration hadoopConf = HadoopFileSystem.getHadoopConfiguration();
+ for (Map.Entry<String, String> e : hadoopConf) {
+ jobConf.set(e.getKey(), e.getValue());
+ }
+ }
+
+ public static JobContext instantiateJobContext(JobConf jobConf, JobID jobId) throws Exception {
+ try {
+ // for Hadoop 1.xx
+ Class<?> clazz = null;
+ if(!TaskAttemptContext.class.isInterface()) {
+ clazz = Class.forName("org.apache.hadoop.mapred.JobContext", true, Thread.currentThread().getContextClassLoader());
+ }
+ // for Hadoop 2.xx
+ else {
+ clazz = Class.forName("org.apache.hadoop.mapred.JobContextImpl", true, Thread.currentThread().getContextClassLoader());
+ }
+ Constructor<?> constructor = clazz.getDeclaredConstructor(JobConf.class, org.apache.hadoop.mapreduce.JobID.class);
+ // for Hadoop 1.xx
+ constructor.setAccessible(true);
+ JobContext context = (JobContext) constructor.newInstance(jobConf, jobId);
+
+ return context;
+ } catch(Exception e) {
+ throw new Exception("Could not create instance of JobContext.", e);
+ }
+ }
+
+ public static TaskAttemptContext instantiateTaskAttemptContext(JobConf jobConf, TaskAttemptID taskAttemptID) throws Exception {
+ try {
+ // for Hadoop 1.xx
+ Class<?> clazz = null;
+ if(!TaskAttemptContext.class.isInterface()) {
+ clazz = Class.forName("org.apache.hadoop.mapred.TaskAttemptContext", true, Thread.currentThread().getContextClassLoader());
+ }
+ // for Hadoop 2.xx
+ else {
+ clazz = Class.forName("org.apache.hadoop.mapred.TaskAttemptContextImpl", true, Thread.currentThread().getContextClassLoader());
+ }
+ Constructor<?> constructor = clazz.getDeclaredConstructor(JobConf.class, TaskAttemptID.class);
+ // for Hadoop 1.xx
+ constructor.setAccessible(true);
+ TaskAttemptContext context = (TaskAttemptContext) constructor.newInstance(jobConf, taskAttemptID);
+ return context;
+ } catch(Exception e) {
+ throw new Exception("Could not create instance of TaskAttemptContext.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyProgressable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyProgressable.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyProgressable.java
new file mode 100644
index 0000000..483dd2f
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyProgressable.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.wrapper;
+
+import org.apache.hadoop.util.Progressable;
+
+/**
+ * This is a dummy progress
+ *
+ */
+public class HadoopDummyProgressable implements Progressable {
+ @Override
+ public void progress() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
new file mode 100644
index 0000000..84a1e9e
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.wrapper;
+
+import org.apache.hadoop.mapred.Counters.Counter;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * This is a dummy progress monitor / reporter
+ *
+ */
+public class HadoopDummyReporter implements Reporter {
+
+ @Override
+ public void progress() {
+ }
+
+ @Override
+ public void setStatus(String status) {
+
+ }
+
+ @Override
+ public Counter getCounter(Enum<?> name) {
+ return null;
+ }
+
+ @Override
+ public Counter getCounter(String group, String name) {
+ return null;
+ }
+
+ @Override
+ public void incrCounter(Enum<?> key, long amount) {
+
+ }
+
+ @Override
+ public void incrCounter(String group, String counter, long amount) {
+
+ }
+
+ @Override
+ public InputSplit getInputSplit() throws UnsupportedOperationException {
+ return null;
+ }
+ // There should be an @Override, but some CDH4 dependency does not contain this method
+ public float getProgress() {
+ return 0;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopInputSplit.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopInputSplit.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopInputSplit.java
new file mode 100644
index 0000000..aa2155d
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopInputSplit.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapred.wrapper;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+import org.apache.flink.core.io.LocatableInputSplit;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.io.WritableFactories;
+import org.apache.hadoop.mapred.JobConf;
+
+
+public class HadoopInputSplit extends LocatableInputSplit {
+
+ private static final long serialVersionUID = 1L;
+
+
+ private transient org.apache.hadoop.mapred.InputSplit hadoopInputSplit;
+
+ private JobConf jobConf;
+
+ private int splitNumber;
+ private String hadoopInputSplitTypeName;
+
+
+ public org.apache.hadoop.mapred.InputSplit getHadoopInputSplit() {
+ return hadoopInputSplit;
+ }
+
+ public HadoopInputSplit() {
+ super();
+ }
+
+ public HadoopInputSplit(int splitNumber, org.apache.hadoop.mapred.InputSplit hInputSplit, JobConf jobconf) {
+
+ this.splitNumber = splitNumber;
+ this.hadoopInputSplit = hInputSplit;
+ this.hadoopInputSplitTypeName = hInputSplit.getClass().getName();
+ this.jobConf = jobconf;
+
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ out.writeInt(splitNumber);
+ out.writeUTF(hadoopInputSplitTypeName);
+ jobConf.write(out);
+ hadoopInputSplit.write(out);
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+ this.splitNumber = in.readInt();
+ this.hadoopInputSplitTypeName = in.readUTF();
+ if(hadoopInputSplit == null) {
+ try {
+ Class<? extends org.apache.hadoop.io.Writable> inputSplit =
+ Class.forName(hadoopInputSplitTypeName).asSubclass(org.apache.hadoop.io.Writable.class);
+ this.hadoopInputSplit = (org.apache.hadoop.mapred.InputSplit) WritableFactories.newInstance( inputSplit );
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Unable to create InputSplit", e);
+ }
+ }
+ jobConf = new JobConf();
+ jobConf.readFields(in);
+ if (this.hadoopInputSplit instanceof Configurable) {
+ ((Configurable) this.hadoopInputSplit).setConf(this.jobConf);
+ }
+ this.hadoopInputSplit.readFields(in);
+
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeInt(splitNumber);
+ out.writeUTF(hadoopInputSplitTypeName);
+ jobConf.write(out);
+ hadoopInputSplit.write(out);
+
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ this.splitNumber=in.readInt();
+ this.hadoopInputSplitTypeName = in.readUTF();
+ if(hadoopInputSplit == null) {
+ try {
+ Class<? extends org.apache.hadoop.io.Writable> inputSplit =
+ Class.forName(hadoopInputSplitTypeName).asSubclass(org.apache.hadoop.io.Writable.class);
+ this.hadoopInputSplit = (org.apache.hadoop.mapred.InputSplit) WritableFactories.newInstance( inputSplit );
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Unable to create InputSplit", e);
+ }
+ }
+ jobConf = new JobConf();
+ jobConf.readFields(in);
+ if (this.hadoopInputSplit instanceof Configurable) {
+ ((Configurable) this.hadoopInputSplit).setConf(this.jobConf);
+ }
+ this.hadoopInputSplit.readFields(in);
+ }
+
+ @Override
+ public int getSplitNumber() {
+ return this.splitNumber;
+ }
+
+ @Override
+ public String[] getHostnames() {
+ try {
+ return this.hadoopInputSplit.getLocations();
+ } catch(IOException ioe) {
+ return new String[0];
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopOutputCollector.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopOutputCollector.java
new file mode 100644
index 0000000..fcb6841
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopOutputCollector.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapred.wrapper;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Collector;
+import org.apache.hadoop.mapred.OutputCollector;
+
+import java.io.IOException;
+
+/**
+ * A Hadoop OutputCollector that wraps a Flink OutputCollector.
+ * On each call of collect() the data is forwarded to the wrapped Flink collector.
+ *
+ */
+@SuppressWarnings("rawtypes")
+public final class HadoopOutputCollector<KEY,VALUE>
+ implements OutputCollector<KEY,VALUE> {
+
+ private Collector<Tuple2<KEY,VALUE>> flinkCollector;
+
+ private final Tuple2<KEY,VALUE> outTuple = new Tuple2<KEY, VALUE>();
+
+ /**
+ * Set the wrapped Flink collector.
+ *
+ * @param flinkCollector The wrapped Flink OutputCollector.
+ */
+ public void setFlinkCollector(Collector<Tuple2<KEY, VALUE>> flinkCollector) {
+ this.flinkCollector = flinkCollector;
+ }
+
+ /**
+ * Use the wrapped Flink collector to collect a key-value pair for Flink.
+ *
+ * @param key the key to collect
+ * @param val the value to collect
+ * @throws IOException unexpected of key or value in key-value pair.
+ */
+ @Override
+ public void collect(final KEY key, final VALUE val) throws IOException {
+
+ this.outTuple.f0 = key;
+ this.outTuple.f1 = val;
+ this.flinkCollector.collect(outTuple);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIterator.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIterator.java
new file mode 100644
index 0000000..5ecac2e
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIterator.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapred.wrapper;
+
+import java.util.Iterator;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.operators.translation.TupleUnwrappingIterator;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+
+/**
+ * Wraps a Flink Tuple2 (key-value-pair) iterator into an iterator over the second (value) field.
+ */
+@SuppressWarnings("rawtypes")
+public class HadoopTupleUnwrappingIterator<KEY,VALUE>
+ extends TupleUnwrappingIterator<VALUE, KEY> implements java.io.Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private Iterator<Tuple2<KEY,VALUE>> iterator;
+
+ private final TypeSerializer<KEY> keySerializer;
+
+ private boolean atFirst = false;
+ private KEY curKey = null;
+ private VALUE firstValue = null;
+
+ public HadoopTupleUnwrappingIterator(Class<KEY> keyClass) {
+ this.keySerializer = TypeExtractor.getForClass((Class<KEY>) keyClass).createSerializer();
+ }
+
+ /**
+ * Set the Flink iterator to wrap.
+ *
+ * @param iterator The Flink iterator to wrap.
+ */
+ @Override()
+ public void set(final Iterator<Tuple2<KEY,VALUE>> iterator) {
+ this.iterator = iterator;
+ if(this.hasNext()) {
+ final Tuple2<KEY, VALUE> tuple = iterator.next();
+ this.curKey = keySerializer.copy(tuple.f0);
+ this.firstValue = tuple.f1;
+ this.atFirst = true;
+ } else {
+ this.atFirst = false;
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ if(this.atFirst) {
+ return true;
+ }
+ return iterator.hasNext();
+ }
+
+ @Override
+ public VALUE next() {
+ if(this.atFirst) {
+ this.atFirst = false;
+ return firstValue;
+ }
+
+ final Tuple2<KEY, VALUE> tuple = iterator.next();
+ return tuple.f1;
+ }
+
+ public KEY getCurrentKey() {
+ return this.curKey;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopInputFormat.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopInputFormat.java
new file mode 100644
index 0000000..20006b8
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopInputFormat.java
@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapreduce;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.api.common.io.LocatableInputSplitAssigner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
+import org.apache.flink.api.common.io.statistics.BaseStatistics;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.io.InputSplitAssigner;
+import org.apache.flink.hadoopcompatibility.mapreduce.utils.HadoopUtils;
+import org.apache.flink.hadoopcompatibility.mapreduce.wrapper.HadoopInputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+
+public class HadoopInputFormat<K, V> implements InputFormat<Tuple2<K,V>, HadoopInputSplit>, ResultTypeQueryable<Tuple2<K,V>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(HadoopInputFormat.class);
+
+ private org.apache.hadoop.mapreduce.InputFormat<K, V> mapreduceInputFormat;
+ private Class<K> keyClass;
+ private Class<V> valueClass;
+ private org.apache.hadoop.conf.Configuration configuration;
+
+ private transient RecordReader<K, V> recordReader;
+ private boolean fetched = false;
+ private boolean hasNext;
+
+ public HadoopInputFormat() {
+ super();
+ }
+
+ public HadoopInputFormat(org.apache.hadoop.mapreduce.InputFormat<K,V> mapreduceInputFormat, Class<K> key, Class<V> value, Job job) {
+ super();
+ this.mapreduceInputFormat = mapreduceInputFormat;
+ this.keyClass = key;
+ this.valueClass = value;
+ this.configuration = job.getConfiguration();
+ HadoopUtils.mergeHadoopConf(configuration);
+ }
+
+ public void setConfiguration(org.apache.hadoop.conf.Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ public org.apache.hadoop.mapreduce.InputFormat<K,V> getHadoopInputFormat() {
+ return this.mapreduceInputFormat;
+ }
+
+ public void setHadoopInputFormat(org.apache.hadoop.mapreduce.InputFormat<K,V> mapreduceInputFormat) {
+ this.mapreduceInputFormat = mapreduceInputFormat;
+ }
+
+ public org.apache.hadoop.conf.Configuration getConfiguration() {
+ return this.configuration;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // InputFormat
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void configure(Configuration parameters) {
+ // nothing to do
+ }
+
+ @Override
+ public BaseStatistics getStatistics(BaseStatistics cachedStats) throws IOException {
+ // only gather base statistics for FileInputFormats
+ if(!(mapreduceInputFormat instanceof FileInputFormat)) {
+ return null;
+ }
+
+ JobContext jobContext = null;
+ try {
+ jobContext = HadoopUtils.instantiateJobContext(configuration, null);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ final FileBaseStatistics cachedFileStats = (cachedStats != null && cachedStats instanceof FileBaseStatistics) ?
+ (FileBaseStatistics) cachedStats : null;
+
+ try {
+ final org.apache.hadoop.fs.Path[] paths = FileInputFormat.getInputPaths(jobContext);
+ return getFileStats(cachedFileStats, paths, new ArrayList<FileStatus>(1));
+ } catch (IOException ioex) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn("Could not determine statistics due to an io error: "
+ + ioex.getMessage());
+ }
+ } catch (Throwable t) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Unexpected problem while getting the file statistics: "
+ + t.getMessage(), t);
+ }
+ }
+
+ // no statistics available
+ return null;
+ }
+
+ @Override
+ public HadoopInputSplit[] createInputSplits(int minNumSplits)
+ throws IOException {
+ configuration.setInt("mapreduce.input.fileinputformat.split.minsize", minNumSplits);
+
+ JobContext jobContext = null;
+ try {
+ jobContext = HadoopUtils.instantiateJobContext(configuration, new JobID());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ List<org.apache.hadoop.mapreduce.InputSplit> splits;
+ try {
+ splits = this.mapreduceInputFormat.getSplits(jobContext);
+ } catch (InterruptedException e) {
+ throw new IOException("Could not get Splits.", e);
+ }
+ HadoopInputSplit[] hadoopInputSplits = new HadoopInputSplit[splits.size()];
+
+ for(int i = 0; i < hadoopInputSplits.length; i++){
+ hadoopInputSplits[i] = new HadoopInputSplit(i, splits.get(i), jobContext);
+ }
+ return hadoopInputSplits;
+ }
+
+ @Override
+ public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
+ return new LocatableInputSplitAssigner(inputSplits);
+ }
+
+ @Override
+ public void open(HadoopInputSplit split) throws IOException {
+ TaskAttemptContext context = null;
+ try {
+ context = HadoopUtils.instantiateTaskAttemptContext(configuration, new TaskAttemptID());
+ } catch(Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ try {
+ this.recordReader = this.mapreduceInputFormat
+ .createRecordReader(split.getHadoopInputSplit(), context);
+ this.recordReader.initialize(split.getHadoopInputSplit(), context);
+ } catch (InterruptedException e) {
+ throw new IOException("Could not create RecordReader.", e);
+ } finally {
+ this.fetched = false;
+ }
+ }
+
+ @Override
+ public boolean reachedEnd() throws IOException {
+ if(!this.fetched) {
+ fetchNext();
+ }
+ return !this.hasNext;
+ }
+
+ private void fetchNext() throws IOException {
+ try {
+ this.hasNext = this.recordReader.nextKeyValue();
+ } catch (InterruptedException e) {
+ throw new IOException("Could not fetch next KeyValue pair.", e);
+ } finally {
+ this.fetched = true;
+ }
+ }
+
+ @Override
+ public Tuple2<K, V> nextRecord(Tuple2<K, V> record) throws IOException {
+ if(!this.fetched) {
+ fetchNext();
+ }
+ if(!this.hasNext) {
+ return null;
+ }
+ try {
+ record.f0 = this.recordReader.getCurrentKey();
+ record.f1 = this.recordReader.getCurrentValue();
+ } catch (InterruptedException e) {
+ throw new IOException("Could not get KeyValue pair.", e);
+ }
+ this.fetched = false;
+
+ return record;
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.recordReader.close();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Helper methods
+ // --------------------------------------------------------------------------------------------
+
+ private FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, org.apache.hadoop.fs.Path[] hadoopFilePaths,
+ ArrayList<FileStatus> files) throws IOException {
+
+ long latestModTime = 0L;
+
+ // get the file info and check whether the cached statistics are still valid.
+ for(org.apache.hadoop.fs.Path hadoopPath : hadoopFilePaths) {
+
+ final Path filePath = new Path(hadoopPath.toUri());
+ final FileSystem fs = FileSystem.get(filePath.toUri());
+
+ final FileStatus file = fs.getFileStatus(filePath);
+ latestModTime = Math.max(latestModTime, file.getModificationTime());
+
+ // enumerate all files and check their modification time stamp.
+ if (file.isDir()) {
+ FileStatus[] fss = fs.listStatus(filePath);
+ files.ensureCapacity(files.size() + fss.length);
+
+ for (FileStatus s : fss) {
+ if (!s.isDir()) {
+ files.add(s);
+ latestModTime = Math.max(s.getModificationTime(), latestModTime);
+ }
+ }
+ } else {
+ files.add(file);
+ }
+ }
+
+ // check whether the cached statistics are still valid, if we have any
+ if (cachedStats != null && latestModTime <= cachedStats.getLastModificationTime()) {
+ return cachedStats;
+ }
+
+ // calculate the whole length
+ long len = 0;
+ for (FileStatus s : files) {
+ len += s.getLen();
+ }
+
+ // sanity check
+ if (len <= 0) {
+ len = BaseStatistics.SIZE_UNKNOWN;
+ }
+
+ return new FileBaseStatistics(latestModTime, len, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Custom serialization methods
+ // --------------------------------------------------------------------------------------------
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeUTF(this.mapreduceInputFormat.getClass().getName());
+ out.writeUTF(this.keyClass.getName());
+ out.writeUTF(this.valueClass.getName());
+ this.configuration.write(out);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ String hadoopInputFormatClassName = in.readUTF();
+ String keyClassName = in.readUTF();
+ String valueClassName = in.readUTF();
+
+ org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
+ configuration.readFields(in);
+
+ if(this.configuration == null) {
+ this.configuration = configuration;
+ }
+
+ try {
+ this.mapreduceInputFormat = (org.apache.hadoop.mapreduce.InputFormat<K,V>) Class.forName(hadoopInputFormatClassName, true, Thread.currentThread().getContextClassLoader()).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to instantiate the hadoop input format", e);
+ }
+ try {
+ this.keyClass = (Class<K>) Class.forName(keyClassName, true, Thread.currentThread().getContextClassLoader());
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to find key class.", e);
+ }
+ try {
+ this.valueClass = (Class<V>) Class.forName(valueClassName, true, Thread.currentThread().getContextClassLoader());
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to find value class.", e);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // ResultTypeQueryable
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public TypeInformation<Tuple2<K,V>> getProducedType() {
+ return new TupleTypeInfo<Tuple2<K,V>>(TypeExtractor.createTypeInfo(keyClass), TypeExtractor.createTypeInfo(valueClass));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopOutputFormat.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopOutputFormat.java
new file mode 100644
index 0000000..696e1be
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/HadoopOutputFormat.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapreduce;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+import org.apache.flink.api.common.io.FinalizeOnMaster;
+import org.apache.flink.api.common.io.OutputFormat;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.hadoopcompatibility.mapreduce.utils.HadoopUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+
+
+public class HadoopOutputFormat<K,V> implements OutputFormat<Tuple2<K,V>>, FinalizeOnMaster {
+
+ private static final long serialVersionUID = 1L;
+
+ private org.apache.hadoop.conf.Configuration configuration;
+ private org.apache.hadoop.mapreduce.OutputFormat<K,V> mapreduceOutputFormat;
+ private transient RecordWriter<K,V> recordWriter;
+ private transient FileOutputCommitter fileOutputCommitter;
+ private transient TaskAttemptContext context;
+ private transient int taskNumber;
+
+ public HadoopOutputFormat(org.apache.hadoop.mapreduce.OutputFormat<K,V> mapreduceOutputFormat, Job job) {
+ super();
+ this.mapreduceOutputFormat = mapreduceOutputFormat;
+ this.configuration = job.getConfiguration();
+ HadoopUtils.mergeHadoopConf(configuration);
+ }
+
+ public void setConfiguration(org.apache.hadoop.conf.Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ public org.apache.hadoop.conf.Configuration getConfiguration() {
+ return this.configuration;
+ }
+
+ public org.apache.hadoop.mapreduce.OutputFormat<K,V> getHadoopOutputFormat() {
+ return this.mapreduceOutputFormat;
+ }
+
+ public void setHadoopOutputFormat(org.apache.hadoop.mapreduce.OutputFormat<K,V> mapreduceOutputFormat) {
+ this.mapreduceOutputFormat = mapreduceOutputFormat;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // OutputFormat
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void configure(Configuration parameters) {
+ // nothing to do
+ }
+
+ /**
+ * create the temporary output file for hadoop RecordWriter.
+ * @param taskNumber The number of the parallel instance.
+ * @param numTasks The number of parallel tasks.
+ * @throws IOException
+ */
+ @Override
+ public void open(int taskNumber, int numTasks) throws IOException {
+ if (Integer.toString(taskNumber + 1).length() > 6) {
+ throw new IOException("Task id too large.");
+ }
+
+ this.taskNumber = taskNumber+1;
+
+ // for hadoop 2.2
+ this.configuration.set("mapreduce.output.basename", "tmp");
+
+ TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_"
+ + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s"," ").replace(" ", "0")
+ + Integer.toString(taskNumber + 1)
+ + "_0");
+
+ this.configuration.set("mapred.task.id", taskAttemptID.toString());
+ this.configuration.setInt("mapred.task.partition", taskNumber + 1);
+ // for hadoop 2.2
+ this.configuration.set("mapreduce.task.attempt.id", taskAttemptID.toString());
+ this.configuration.setInt("mapreduce.task.partition", taskNumber + 1);
+
+ try {
+ this.context = HadoopUtils.instantiateTaskAttemptContext(this.configuration, taskAttemptID);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ this.fileOutputCommitter = new FileOutputCommitter(new Path(this.configuration.get("mapred.output.dir")), context);
+
+ try {
+ this.fileOutputCommitter.setupJob(HadoopUtils.instantiateJobContext(this.configuration, new JobID()));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ // compatible for hadoop 2.2.0, the temporary output directory is different from hadoop 1.2.1
+ this.configuration.set("mapreduce.task.output.dir", this.fileOutputCommitter.getWorkPath().toString());
+
+ try {
+ this.recordWriter = this.mapreduceOutputFormat.getRecordWriter(this.context);
+ } catch (InterruptedException e) {
+ throw new IOException("Could not create RecordWriter.", e);
+ }
+ }
+
+
+ @Override
+ public void writeRecord(Tuple2<K, V> record) throws IOException {
+ try {
+ this.recordWriter.write(record.f0, record.f1);
+ } catch (InterruptedException e) {
+ throw new IOException("Could not write Record.", e);
+ }
+ }
+
+ /**
+ * commit the task by moving the output file out from the temporary directory.
+ * @throws IOException
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ this.recordWriter.close(this.context);
+ } catch (InterruptedException e) {
+ throw new IOException("Could not close RecordReader.", e);
+ }
+
+ if (this.fileOutputCommitter.needsTaskCommit(this.context)) {
+ this.fileOutputCommitter.commitTask(this.context);
+ }
+
+ Path outputPath = new Path(this.configuration.get("mapred.output.dir"));
+
+ // rename tmp-file to final name
+ FileSystem fs = FileSystem.get(outputPath.toUri(), this.configuration);
+
+ String taskNumberStr = Integer.toString(this.taskNumber);
+ String tmpFileTemplate = "tmp-r-00000";
+ String tmpFile = tmpFileTemplate.substring(0,11-taskNumberStr.length())+taskNumberStr;
+
+ if(fs.exists(new Path(outputPath.toString()+"/"+tmpFile))) {
+ fs.rename(new Path(outputPath.toString()+"/"+tmpFile), new Path(outputPath.toString()+"/"+taskNumberStr));
+ }
+ }
+
+ @Override
+ public void finalizeGlobal(int parallelism) throws IOException {
+
+ JobContext jobContext;
+ TaskAttemptContext taskContext;
+ try {
+
+ TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_"
+ + String.format("%" + (6 - Integer.toString(1).length()) + "s"," ").replace(" ", "0")
+ + Integer.toString(1)
+ + "_0");
+
+ jobContext = HadoopUtils.instantiateJobContext(this.configuration, new JobID());
+ taskContext = HadoopUtils.instantiateTaskAttemptContext(this.configuration, taskAttemptID);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ this.fileOutputCommitter = new FileOutputCommitter(new Path(this.configuration.get("mapred.output.dir")), taskContext);
+
+ // finalize HDFS output format
+ this.fileOutputCommitter.commitJob(jobContext);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Custom serialization methods
+ // --------------------------------------------------------------------------------------------
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeUTF(this.mapreduceOutputFormat.getClass().getName());
+ this.configuration.write(out);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ String hadoopOutputFormatClassName = in.readUTF();
+
+ org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
+ configuration.readFields(in);
+
+ if(this.configuration == null) {
+ this.configuration = configuration;
+ }
+
+ try {
+ this.mapreduceOutputFormat = (org.apache.hadoop.mapreduce.OutputFormat<K,V>) Class.forName(hadoopOutputFormatClassName, true, Thread.currentThread().getContextClassLoader()).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to instantiate the hadoop output format", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java
new file mode 100644
index 0000000..2b99fd2
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapreduce.example;
+
+import org.apache.flink.api.java.aggregation.Aggregations;
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Collector;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.hadoopcompatibility.mapreduce.HadoopInputFormat;
+import org.apache.flink.hadoopcompatibility.mapreduce.HadoopOutputFormat;
+
+/**
+ * Implements a word count which takes the input file and counts the number of
+ * occurrences of each word in the file and writes the result back to disk.
+ *
+ * This example shows how to use Hadoop Input Formats, how to convert Hadoop Writables to
+ * common Java types for better usage in a Flink job and how to use Hadoop Output Formats.
+ */
+@SuppressWarnings("serial")
+public class WordCount {
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 2) {
+ System.err.println("Usage: WordCount <input path> <result path>");
+ return;
+ }
+
+ final String inputPath = args[0];
+ final String outputPath = args[1];
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ // Set up the Hadoop Input Format
+ Job job = Job.getInstance();
+ HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, job);
+ TextInputFormat.addInputPath(job, new Path(inputPath));
+
+ // Create a Flink job with it
+ DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);
+
+ // Tokenize the line and convert from Writable "Text" to String for better handling
+ DataSet<Tuple2<String, Integer>> words = text.flatMap(new Tokenizer());
+
+ // Sum up the words
+ DataSet<Tuple2<String, Integer>> result = words.groupBy(0).aggregate(Aggregations.SUM, 1);
+
+ // Convert String back to Writable "Text" for use with Hadoop Output Format
+ DataSet<Tuple2<Text, IntWritable>> hadoopResult = result.map(new HadoopDatatypeMapper());
+
+ // Set up Hadoop Output Format
+ HadoopOutputFormat<Text, IntWritable> hadoopOutputFormat = new HadoopOutputFormat<Text, IntWritable>(new TextOutputFormat<Text, IntWritable>(), job);
+ hadoopOutputFormat.getConfiguration().set("mapreduce.output.textoutputformat.separator", " ");
+ hadoopOutputFormat.getConfiguration().set("mapred.textoutputformat.separator", " "); // set the value for both, since this test
+ // is being executed with both types (hadoop1 and hadoop2 profile)
+ TextOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ // Output & Execute
+ hadoopResult.output(hadoopOutputFormat);
+ env.execute("Word Count");
+ }
+
+ /**
+ * Splits a line into words and converts Hadoop Writables into normal Java data types.
+ */
+ public static final class Tokenizer extends RichFlatMapFunction<Tuple2<LongWritable, Text>, Tuple2<String, Integer>> {
+
+ @Override
+ public void flatMap(Tuple2<LongWritable, Text> value, Collector<Tuple2<String, Integer>> out) {
+ // normalize and split the line
+ String line = value.f1.toString();
+ String[] tokens = line.toLowerCase().split("\\W+");
+
+ // emit the pairs
+ for (String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new Tuple2<String, Integer>(token, 1));
+ }
+ }
+ }
+ }
+
+ /**
+ * Converts Java data types to Hadoop Writables.
+ */
+ public static final class HadoopDatatypeMapper extends RichMapFunction<Tuple2<String, Integer>, Tuple2<Text, IntWritable>> {
+
+ @Override
+ public Tuple2<Text, IntWritable> map(Tuple2<String, Integer> value) throws Exception {
+ return new Tuple2<Text, IntWritable>(new Text(value.f0), new IntWritable(value.f1));
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/utils/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/utils/HadoopUtils.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/utils/HadoopUtils.java
new file mode 100644
index 0000000..86b730f
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/utils/HadoopUtils.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.hadoopcompatibility.mapreduce.utils;
+
+import java.lang.reflect.Constructor;
+import java.util.Map;
+
+import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+
+public class HadoopUtils {
+
+ /**
+ * Merge HadoopConfiguration into Configuration. This is necessary for the HDFS configuration.
+ */
+ public static void mergeHadoopConf(Configuration configuration) {
+ Configuration hadoopConf = HadoopFileSystem.getHadoopConfiguration();
+
+ for (Map.Entry<String, String> e : hadoopConf) {
+ configuration.set(e.getKey(), e.getValue());
+ }
+ }
+
+ public static JobContext instantiateJobContext(Configuration configuration, JobID jobId) throws Exception {
+ try {
+ Class<?> clazz;
+ // for Hadoop 1.xx
+ if(JobContext.class.isInterface()) {
+ clazz = Class.forName("org.apache.hadoop.mapreduce.task.JobContextImpl", true, Thread.currentThread().getContextClassLoader());
+ }
+ // for Hadoop 2.xx
+ else {
+ clazz = Class.forName("org.apache.hadoop.mapreduce.JobContext", true, Thread.currentThread().getContextClassLoader());
+ }
+ Constructor<?> constructor = clazz.getConstructor(Configuration.class, JobID.class);
+ JobContext context = (JobContext) constructor.newInstance(configuration, jobId);
+
+ return context;
+ } catch(Exception e) {
+ throw new Exception("Could not create instance of JobContext.");
+ }
+ }
+
+ public static TaskAttemptContext instantiateTaskAttemptContext(Configuration configuration, TaskAttemptID taskAttemptID) throws Exception {
+ try {
+ Class<?> clazz;
+ // for Hadoop 1.xx
+ if(JobContext.class.isInterface()) {
+ clazz = Class.forName("org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl");
+ }
+ // for Hadoop 2.xx
+ else {
+ clazz = Class.forName("org.apache.hadoop.mapreduce.TaskAttemptContext");
+ }
+ Constructor<?> constructor = clazz.getConstructor(Configuration.class, TaskAttemptID.class);
+ TaskAttemptContext context = (TaskAttemptContext) constructor.newInstance(configuration, taskAttemptID);
+
+ return context;
+ } catch(Exception e) {
+ throw new Exception("Could not create instance of TaskAttemptContext.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/wrapper/HadoopInputSplit.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/wrapper/HadoopInputSplit.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/wrapper/HadoopInputSplit.java
new file mode 100644
index 0000000..7477c28
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/wrapper/HadoopInputSplit.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.hadoopcompatibility.mapreduce.wrapper;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+import org.apache.flink.core.io.LocatableInputSplit;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
+import org.apache.hadoop.mapreduce.JobContext;
+
+
+public class HadoopInputSplit extends LocatableInputSplit {
+
+ public transient org.apache.hadoop.mapreduce.InputSplit mapreduceInputSplit;
+ public transient JobContext jobContext;
+
+ private int splitNumber;
+
+ public org.apache.hadoop.mapreduce.InputSplit getHadoopInputSplit() {
+ return mapreduceInputSplit;
+ }
+
+
+ public HadoopInputSplit() {
+ super();
+ }
+
+
+ public HadoopInputSplit(int splitNumber, org.apache.hadoop.mapreduce.InputSplit mapreduceInputSplit, JobContext jobContext) {
+ this.splitNumber = splitNumber;
+ if(!(mapreduceInputSplit instanceof Writable)) {
+ throw new IllegalArgumentException("InputSplit must implement Writable interface.");
+ }
+ this.mapreduceInputSplit = mapreduceInputSplit;
+ this.jobContext = jobContext;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ out.writeInt(this.splitNumber);
+ out.writeUTF(this.mapreduceInputSplit.getClass().getName());
+ Writable w = (Writable) this.mapreduceInputSplit;
+ w.write(out);
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+ this.splitNumber = in.readInt();
+ String className = in.readUTF();
+
+ if(this.mapreduceInputSplit == null) {
+ try {
+ Class<? extends org.apache.hadoop.io.Writable> inputSplit =
+ Class.forName(className).asSubclass(org.apache.hadoop.io.Writable.class);
+ this.mapreduceInputSplit = (org.apache.hadoop.mapreduce.InputSplit) WritableFactories.newInstance(inputSplit);
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to create InputSplit", e);
+ }
+ }
+ ((Writable)this.mapreduceInputSplit).readFields(in);
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeInt(this.splitNumber);
+ out.writeUTF(this.mapreduceInputSplit.getClass().getName());
+ Writable w = (Writable) this.mapreduceInputSplit;
+ w.write(out);
+
+ }
+
+ @SuppressWarnings("unchecked")
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ this.splitNumber=in.readInt();
+ String className = in.readUTF();
+
+ if(this.mapreduceInputSplit == null) {
+ try {
+ Class<? extends org.apache.hadoop.io.Writable> inputSplit =
+ Class.forName(className).asSubclass(org.apache.hadoop.io.Writable.class);
+ this.mapreduceInputSplit = (org.apache.hadoop.mapreduce.InputSplit) WritableFactories.newInstance(inputSplit);
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to create InputSplit", e);
+ }
+ }
+ ((Writable)this.mapreduceInputSplit).readFields(in);
+ }
+
+ @Override
+ public int getSplitNumber() {
+ return this.splitNumber;
+ }
+
+ @Override
+ public String[] getHostnames() {
+ try {
+ return this.mapreduceInputSplit.getLocations();
+ } catch (IOException e) {
+ return new String[0];
+ } catch (InterruptedException e) {
+ return new String[0];
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopIOFormatsITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopIOFormatsITCase.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopIOFormatsITCase.java
new file mode 100644
index 0000000..32396b8
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopIOFormatsITCase.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.hadoopcompatibility.mapred;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.hadoopcompatibility.mapred.HadoopInputFormat;
+import org.apache.flink.test.util.JavaProgramTestBase;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Collection;
+import java.util.LinkedList;
+
+@RunWith(Parameterized.class)
+public class HadoopIOFormatsITCase extends JavaProgramTestBase {
+
+ private static int NUM_PROGRAMS = 2;
+
+ private int curProgId = config.getInteger("ProgramId", -1);
+ private String[] resultPath;
+ private String[] expectedResult;
+ private String sequenceFileInPath;
+ private String sequenceFileInPathNull;
+
+ public HadoopIOFormatsITCase(Configuration config) {
+ super(config);
+ }
+
+ @Override
+ protected void preSubmit() throws Exception {
+ resultPath = new String[] {getTempDirPath("result0"), getTempDirPath("result1") };
+
+ File sequenceFile = createAndRegisterTempFile("seqFile");
+ sequenceFileInPath = sequenceFile.toURI().toString();
+
+ // Create a sequence file
+ org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
+ FileSystem fs = FileSystem.get(URI.create(sequenceFile.getAbsolutePath()), conf);
+ Path path = new Path(sequenceFile.getAbsolutePath());
+
+ // ------------------ Long / Text Key Value pair: ------------
+ int kvCount = 4;
+
+ LongWritable key = new LongWritable();
+ Text value = new Text();
+ SequenceFile.Writer writer = null;
+ try {
+ writer = SequenceFile.createWriter( fs, conf, path, key.getClass(), value.getClass());
+ for (int i = 0; i < kvCount; i ++) {
+ if(i == 1) {
+ // write key = 0 a bit more often.
+ for(int a = 0;a < 15; a++) {
+ key.set(i);
+ value.set(i+" - somestring");
+ writer.append(key, value);
+ }
+ }
+ key.set(i);
+ value.set(i+" - somestring");
+ writer.append(key, value);
+ }
+ } finally {
+ IOUtils.closeStream(writer);
+ }
+
+
+ // ------------------ Long / Text Key Value pair: ------------
+
+ File sequenceFileNull = createAndRegisterTempFile("seqFileNullKey");
+ sequenceFileInPathNull = sequenceFileNull.toURI().toString();
+ path = new Path(sequenceFileInPathNull);
+
+ LongWritable value1 = new LongWritable();
+ SequenceFile.Writer writer1 = null;
+ try {
+ writer1 = SequenceFile.createWriter( fs, conf, path, NullWritable.class, value1.getClass());
+ for (int i = 0; i < kvCount; i ++) {
+ value1.set(i);
+ writer1.append(NullWritable.get(), value1);
+ }
+ } finally {
+ IOUtils.closeStream(writer1);
+ }
+ }
+
+ @Override
+ protected void testProgram() throws Exception {
+ expectedResult = HadoopIOFormatPrograms.runProgram(curProgId, resultPath, sequenceFileInPath, sequenceFileInPathNull);
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ for(int i = 0; i < resultPath.length; i++) {
+ compareResultsByLinesInMemory(expectedResult[i], resultPath[i]);
+ }
+ }
+
+ @Parameters
+ public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {
+
+ LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
+
+ for(int i=1; i <= NUM_PROGRAMS; i++) {
+ Configuration config = new Configuration();
+ config.setInteger("ProgramId", i);
+ tConfigs.add(config);
+ }
+
+ return toParameterList(tConfigs);
+ }
+
+ public static class HadoopIOFormatPrograms {
+
+ public static String[] runProgram(int progId, String resultPath[], String sequenceFileInPath, String sequenceFileInPathNull) throws Exception {
+
+ switch(progId) {
+ case 1: {
+ /**
+ * Test sequence file, including a key access.
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ SequenceFileInputFormat<LongWritable, Text> sfif = new SequenceFileInputFormat<LongWritable, Text>();
+ JobConf hdconf = new JobConf();
+ SequenceFileInputFormat.addInputPath(hdconf, new Path(sequenceFileInPath));
+ HadoopInputFormat<LongWritable, Text> hif = new HadoopInputFormat<LongWritable, Text>(sfif, LongWritable.class, Text.class, hdconf);
+ DataSet<Tuple2<LongWritable, Text>> ds = env.createInput(hif);
+ DataSet<Tuple2<Long, Text>> sumed = ds.map(new MapFunction<Tuple2<LongWritable, Text>, Tuple2<Long, Text>>() {
+ @Override
+ public Tuple2<Long, Text> map(Tuple2<LongWritable, Text> value) throws Exception {
+ return new Tuple2<Long, Text>(value.f0.get(), value.f1);
+ }
+ }).sum(0);
+ sumed.writeAsText(resultPath[0]);
+ DataSet<String> res = ds.distinct(0).map(new MapFunction<Tuple2<LongWritable, Text>, String>() {
+ @Override
+ public String map(Tuple2<LongWritable, Text> value) throws Exception {
+ return value.f1 + " - " + value.f0.get();
+ }
+ });
+ res.writeAsText(resultPath[1]);
+ env.execute();
+
+ // return expected result
+ return new String [] {"(21,3 - somestring)", "0 - somestring - 0\n" +
+ "1 - somestring - 1\n" +
+ "2 - somestring - 2\n" +
+ "3 - somestring - 3\n"};
+
+ }
+ case 2: {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ SequenceFileInputFormat<NullWritable, LongWritable> sfif = new SequenceFileInputFormat<NullWritable, LongWritable>();
+ JobConf hdconf = new JobConf();
+ SequenceFileInputFormat.addInputPath(hdconf, new Path(sequenceFileInPathNull));
+ HadoopInputFormat<NullWritable, LongWritable> hif = new HadoopInputFormat<NullWritable, LongWritable>(sfif, NullWritable.class, LongWritable.class, hdconf);
+ DataSet<Tuple2<NullWritable, LongWritable>> ds = env.createInput(hif);
+ DataSet<Tuple2<Void, Long>> res = ds.map(new MapFunction<Tuple2<NullWritable, LongWritable>, Tuple2<Void, Long>>() {
+ @Override
+ public Tuple2<Void, Long> map(Tuple2<NullWritable, LongWritable> value) throws Exception {
+ return new Tuple2<Void, Long>(null, value.f1.get());
+ }
+ });
+ DataSet<Tuple2<Void, Long>> res1 = res.groupBy(1).sum(1);
+ res1.writeAsText(resultPath[1]);
+ res.writeAsText(resultPath[0]);
+ env.execute();
+
+ // return expected result
+ return new String [] {"(null,2)\n" +
+ "(null,0)\n" +
+ "(null,1)\n" +
+ "(null,3)",
+ "(null,0)\n" +
+ "(null,1)\n" +
+ "(null,2)\n" +
+ "(null,3)"};
+ }
+ default:
+ throw new IllegalArgumentException("Invalid program id");
+ }
+
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopInputFormatTest.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopInputFormatTest.java
new file mode 100644
index 0000000..00fd1f9
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopInputFormatTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.test.hadoopcompatibility.mapred;
+
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.hadoopcompatibility.mapred.HadoopInputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.fail;
+
+
+public class HadoopInputFormatTest {
+
+
+ public class DummyVoidKeyInputFormat<T> extends FileInputFormat<Void, T> {
+
+ public DummyVoidKeyInputFormat() {
+ }
+
+ @Override
+ public org.apache.hadoop.mapred.RecordReader<Void, T> getRecordReader(org.apache.hadoop.mapred.InputSplit inputSplit, JobConf jobConf, Reporter reporter) throws IOException {
+ return null;
+ }
+ }
+
+
+ @Test
+ public void checkTypeInformation() {
+ try {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ // Set up the Hadoop Input Format
+ Job job = Job.getInstance();
+ HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<Void, Long>( new DummyVoidKeyInputFormat(), Void.class, Long.class, new JobConf());
+
+ TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
+ TypeInformation<Tuple2<Void,Long>> testTupleType = new TupleTypeInfo<Tuple2<Void,Long>>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
+
+ if(tupleType.isTupleType()) {
+ if(!((TupleTypeInfo)tupleType).equals(testTupleType)) {
+ fail("Tuple type information was not set correctly!");
+ }
+ } else {
+ fail("Type information was not set to tuple type information!");
+ }
+
+ }
+ catch (Exception ex) {
+ fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapFunctionITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapFunctionITCase.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapFunctionITCase.java
new file mode 100644
index 0000000..850e799
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapFunctionITCase.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.hadoopcompatibility.mapred;
+
+import java.io.IOException;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.hadoopcompatibility.mapred.HadoopMapFunction;
+import org.apache.flink.test.util.MultipleProgramsTestBase;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class HadoopMapFunctionITCase extends MultipleProgramsTestBase {
+
+ public HadoopMapFunctionITCase(ExecutionMode mode){
+ super(mode);
+ }
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ @Test
+ public void testNonPassingMapper() throws Exception{
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
+ DataSet<Tuple2<IntWritable, Text>> nonPassingFlatMapDs = ds.
+ flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new NonPassingMapper()));
+
+ String resultPath = tempFolder.newFile().toURI().toString();
+
+ nonPassingFlatMapDs.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
+ env.execute();
+
+ compareResultsByLinesInMemory("\n", resultPath);
+ }
+
+ @Test
+ public void testDataDuplicatingMapper() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
+ DataSet<Tuple2<IntWritable, Text>> duplicatingFlatMapDs = ds.
+ flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new DuplicatingMapper()));
+
+ String resultPath = tempFolder.newFile().toURI().toString();
+
+ duplicatingFlatMapDs.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
+ env.execute();
+
+ String expected = "(1,Hi)\n" + "(1,HI)\n" +
+ "(2,Hello)\n" + "(2,HELLO)\n" +
+ "(3,Hello world)\n" + "(3,HELLO WORLD)\n" +
+ "(4,Hello world, how are you?)\n" + "(4,HELLO WORLD, HOW ARE YOU?)\n" +
+ "(5,I am fine.)\n" + "(5,I AM FINE.)\n" +
+ "(6,Luke Skywalker)\n" + "(6,LUKE SKYWALKER)\n" +
+ "(7,Comment#1)\n" + "(7,COMMENT#1)\n" +
+ "(8,Comment#2)\n" + "(8,COMMENT#2)\n" +
+ "(9,Comment#3)\n" + "(9,COMMENT#3)\n" +
+ "(10,Comment#4)\n" + "(10,COMMENT#4)\n" +
+ "(11,Comment#5)\n" + "(11,COMMENT#5)\n" +
+ "(12,Comment#6)\n" + "(12,COMMENT#6)\n" +
+ "(13,Comment#7)\n" + "(13,COMMENT#7)\n" +
+ "(14,Comment#8)\n" + "(14,COMMENT#8)\n" +
+ "(15,Comment#9)\n" + "(15,COMMENT#9)\n" +
+ "(16,Comment#10)\n" + "(16,COMMENT#10)\n" +
+ "(17,Comment#11)\n" + "(17,COMMENT#11)\n" +
+ "(18,Comment#12)\n" + "(18,COMMENT#12)\n" +
+ "(19,Comment#13)\n" + "(19,COMMENT#13)\n" +
+ "(20,Comment#14)\n" + "(20,COMMENT#14)\n" +
+ "(21,Comment#15)\n" + "(21,COMMENT#15)\n";
+
+ compareResultsByLinesInMemory(expected, resultPath);
+ }
+
+ @Test
+ public void testConfigurableMapper() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ JobConf conf = new JobConf();
+ conf.set("my.filterPrefix", "Hello");
+
+ DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
+ DataSet<Tuple2<IntWritable, Text>> hellos = ds.
+ flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new ConfigurableMapper(), conf));
+
+ String resultPath = tempFolder.newFile().toURI().toString();
+
+ hellos.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
+ env.execute();
+
+ String expected = "(2,Hello)\n" +
+ "(3,Hello world)\n" +
+ "(4,Hello world, how are you?)\n";
+
+ compareResultsByLinesInMemory(expected, resultPath);
+ }
+
+
+
+ public static class NonPassingMapper implements Mapper<IntWritable, Text, IntWritable, Text> {
+
+ @Override
+ public void map(final IntWritable k, final Text v,
+ final OutputCollector<IntWritable, Text> out, final Reporter r) throws IOException {
+ if ( v.toString().contains("bananas") ) {
+ out.collect(k,v);
+ }
+ }
+
+ @Override
+ public void configure(final JobConf arg0) { }
+
+ @Override
+ public void close() throws IOException { }
+ }
+
+ public static class DuplicatingMapper implements Mapper<IntWritable, Text, IntWritable, Text> {
+
+ @Override
+ public void map(final IntWritable k, final Text v,
+ final OutputCollector<IntWritable, Text> out, final Reporter r) throws IOException {
+ out.collect(k, v);
+ out.collect(k, new Text(v.toString().toUpperCase()));
+ }
+
+ @Override
+ public void configure(final JobConf arg0) { }
+
+ @Override
+ public void close() throws IOException { }
+ }
+
+ public static class ConfigurableMapper implements Mapper<IntWritable, Text, IntWritable, Text> {
+ private String filterPrefix;
+
+ @Override
+ public void map(IntWritable k, Text v, OutputCollector<IntWritable, Text> out, Reporter r)
+ throws IOException {
+ if(v.toString().startsWith(filterPrefix)) {
+ out.collect(k, v);
+ }
+ }
+
+ @Override
+ public void configure(JobConf c) {
+ filterPrefix = c.get("my.filterPrefix");
+ }
+
+ @Override
+ public void close() throws IOException { }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapredITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapredITCase.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapredITCase.java
new file mode 100644
index 0000000..b6650d2
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapredITCase.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.hadoopcompatibility.mapred;
+
+import org.apache.flink.hadoopcompatibility.mapred.example.HadoopMapredCompatWordCount;
+import org.apache.flink.test.testdata.WordCountData;
+import org.apache.flink.test.util.JavaProgramTestBase;
+
+public class HadoopMapredITCase extends JavaProgramTestBase {
+
+ protected String textPath;
+ protected String resultPath;
+
+ @Override
+ protected void preSubmit() throws Exception {
+ textPath = createTempFile("text.txt", WordCountData.TEXT);
+ resultPath = getTempDirPath("result");
+ this.setDegreeOfParallelism(4);
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ compareResultsByLinesInMemory(WordCountData.COUNTS, resultPath, new String[]{".", "_"});
+ }
+
+ @Override
+ protected void testProgram() throws Exception {
+ HadoopMapredCompatWordCount.main(new String[] { textPath, resultPath });
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceCombineFunctionITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceCombineFunctionITCase.java b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceCombineFunctionITCase.java
new file mode 100644
index 0000000..92b0dc3
--- /dev/null
+++ b/flink-staging/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceCombineFunctionITCase.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.hadoopcompatibility.mapred;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.hadoopcompatibility.mapred.HadoopReduceCombineFunction;
+import org.apache.flink.hadoopcompatibility.mapred.HadoopReduceFunction;
+import org.apache.flink.test.util.MultipleProgramsTestBase;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.hamcrest.core.IsEqual;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class HadoopReduceCombineFunctionITCase extends MultipleProgramsTestBase {
+
+ public HadoopReduceCombineFunctionITCase(ExecutionMode mode){
+ super(mode);
+ }
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ @Test
+ public void testStandardCountingWithCombiner() throws Exception{
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<IntWritable, IntWritable>> ds = HadoopTestData.getKVPairDataSet(env).
+ map(new Mapper1());
+
+ DataSet<Tuple2<IntWritable, IntWritable>> counts = ds.
+ groupBy(0).
+ reduceGroup(new HadoopReduceCombineFunction<IntWritable, IntWritable, IntWritable, IntWritable>(
+ new SumReducer(), new SumReducer()));
+
+ String resultPath = tempFolder.newFile().toURI().toString();
+
+ counts.writeAsText(resultPath);
+ env.execute();
+
+ String expected = "(0,5)\n"+
+ "(1,6)\n" +
+ "(2,6)\n" +
+ "(3,4)\n";
+
+ compareResultsByLinesInMemory(expected, resultPath);
+ }
+
+ @Test
+ public void testUngroupedHadoopReducer() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<IntWritable, IntWritable>> ds = HadoopTestData.getKVPairDataSet(env).
+ map(new Mapper2());
+
+ DataSet<Tuple2<IntWritable, IntWritable>> sum = ds.
+ reduceGroup(new HadoopReduceCombineFunction<IntWritable, IntWritable, IntWritable, IntWritable>(
+ new SumReducer(), new SumReducer()));
+
+ String resultPath = tempFolder.newFile().toURI().toString();
+
+ sum.writeAsText(resultPath);
+ env.execute();
+
+ String expected = "(0,231)\n";
+
+ compareResultsByLinesInMemory(expected, resultPath);
+ }
+
+ @Test
+ public void testCombiner() throws Exception {
+ org.junit.Assume.assumeThat(mode, new IsEqual<ExecutionMode>(ExecutionMode.CLUSTER));
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<IntWritable, IntWritable>> ds = HadoopTestData.getKVPairDataSet(env).
+ map(new Mapper3());
+
+ DataSet<Tuple2<IntWritable, IntWritable>> counts = ds.
+ groupBy(0).
+ reduceGroup(new HadoopReduceCombineFunction<IntWritable, IntWritable, IntWritable, IntWritable>(
+ new SumReducer(), new KeyChangingReducer()));
+
+ String resultPath = tempFolder.newFile().toURI().toString();
+
+ counts.writeAsText(resultPath);
+ env.execute();
+
+ String expected = "(0,5)\n"+
+ "(1,6)\n" +
+ "(2,5)\n" +
+ "(3,5)\n";
+
+ compareResultsByLinesInMemory(expected, resultPath);
+ }
+
+ @Test
+ public void testConfigurationViaJobConf() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ JobConf conf = new JobConf();
+ conf.set("my.cntPrefix", "Hello");
+
+ DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env).
+ map(new Mapper4());
+
+ DataSet<Tuple2<IntWritable, IntWritable>> hellos = ds.
+ groupBy(0).
+ reduceGroup(new HadoopReduceFunction<IntWritable, Text, IntWritable, IntWritable>(
+ new ConfigurableCntReducer(), conf));
+
+ String resultPath = tempFolder.newFile().toURI().toString();
+
+ hellos.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ String expected = "(0,0)\n"+
+ "(1,0)\n" +
+ "(2,1)\n" +
+ "(3,1)\n" +
+ "(4,1)\n";
+
+ compareResultsByLinesInMemory(expected, resultPath);
+ }
+
+ public static class SumReducer implements Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
+
+ @Override
+ public void reduce(IntWritable k, Iterator<IntWritable> v, OutputCollector<IntWritable, IntWritable> out, Reporter r)
+ throws IOException {
+
+ int sum = 0;
+ while(v.hasNext()) {
+ sum += v.next().get();
+ }
+ out.collect(k, new IntWritable(sum));
+ }
+
+ @Override
+ public void configure(JobConf arg0) { }
+
+ @Override
+ public void close() throws IOException { }
+ }
+
+ public static class KeyChangingReducer implements Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
+
+ @Override
+ public void reduce(IntWritable k, Iterator<IntWritable> v, OutputCollector<IntWritable, IntWritable> out, Reporter r)
+ throws IOException {
+ while(v.hasNext()) {
+ out.collect(new IntWritable(k.get() % 4), v.next());
+ }
+ }
+
+ @Override
+ public void configure(JobConf arg0) { }
+
+ @Override
+ public void close() throws IOException { }
+ }
+
+ public static class ConfigurableCntReducer implements Reducer<IntWritable, Text, IntWritable, IntWritable> {
+ private String countPrefix;
+
+ @Override
+ public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r)
+ throws IOException {
+ int commentCnt = 0;
+ while(vs.hasNext()) {
+ String v = vs.next().toString();
+ if(v.startsWith(this.countPrefix)) {
+ commentCnt++;
+ }
+ }
+ out.collect(k, new IntWritable(commentCnt));
+ }
+
+ @Override
+ public void configure(final JobConf c) {
+ this.countPrefix = c.get("my.cntPrefix");
+ }
+
+ @Override
+ public void close() throws IOException { }
+ }
+
+ public static class Mapper1 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable,
+ IntWritable>> {
+ private static final long serialVersionUID = 1L;
+ Tuple2<IntWritable,IntWritable> outT = new Tuple2<IntWritable,IntWritable>();
+ @Override
+ public Tuple2<IntWritable, IntWritable> map(Tuple2<IntWritable, Text> v)
+ throws Exception {
+ outT.f0 = new IntWritable(v.f0.get() / 6);
+ outT.f1 = new IntWritable(1);
+ return outT;
+ }
+ }
+
+ public static class Mapper2 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable,
+ IntWritable>> {
+ private static final long serialVersionUID = 1L;
+ Tuple2<IntWritable,IntWritable> outT = new Tuple2<IntWritable,IntWritable>();
+ @Override
+ public Tuple2<IntWritable, IntWritable> map(Tuple2<IntWritable, Text> v)
+ throws Exception {
+ outT.f0 = new IntWritable(0);
+ outT.f1 = v.f0;
+ return outT;
+ }
+ }
+
+ public static class Mapper3 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable, IntWritable>> {
+ private static final long serialVersionUID = 1L;
+ Tuple2<IntWritable,IntWritable> outT = new Tuple2<IntWritable,IntWritable>();
+ @Override
+ public Tuple2<IntWritable, IntWritable> map(Tuple2<IntWritable, Text> v)
+ throws Exception {
+ outT.f0 = v.f0;
+ outT.f1 = new IntWritable(1);
+ return outT;
+ }
+ }
+
+ public static class Mapper4 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable, Text>> {
+ private static final long serialVersionUID = 1L;
+ @Override
+ public Tuple2<IntWritable, Text> map(Tuple2<IntWritable, Text> v)
+ throws Exception {
+ v.f0 = new IntWritable(v.f0.get() % 5);
+ return v;
+ }
+ }
+}
[49/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopInputFormat.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopInputFormat.java
deleted file mode 100644
index 326a1c4..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopInputFormat.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.util.ArrayList;
-
-import org.apache.flink.api.common.io.LocatableInputSplitAssigner;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
-import org.apache.flink.api.common.io.statistics.BaseStatistics;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.fs.FileStatus;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.core.io.InputSplitAssigner;
-import org.apache.flink.hadoopcompatibility.mapred.utils.HadoopUtils;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopInputSplit;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.util.ReflectionUtils;
-
-public class HadoopInputFormat<K, V> implements InputFormat<Tuple2<K,V>, HadoopInputSplit>, ResultTypeQueryable<Tuple2<K,V>> {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(HadoopInputFormat.class);
-
- private org.apache.hadoop.mapred.InputFormat<K, V> mapredInputFormat;
- private Class<K> keyClass;
- private Class<V> valueClass;
- private JobConf jobConf;
-
- private transient K key;
- private transient V value;
-
- private transient RecordReader<K, V> recordReader;
- private transient boolean fetched = false;
- private transient boolean hasNext;
-
- public HadoopInputFormat() {
- super();
- }
-
- public HadoopInputFormat(org.apache.hadoop.mapred.InputFormat<K,V> mapredInputFormat, Class<K> key, Class<V> value, JobConf job) {
- super();
- this.mapredInputFormat = mapredInputFormat;
- this.keyClass = key;
- this.valueClass = value;
- HadoopUtils.mergeHadoopConf(job);
- this.jobConf = job;
- ReflectionUtils.setConf(mapredInputFormat, jobConf);
- }
-
- public void setJobConf(JobConf job) {
- this.jobConf = job;
- }
-
- public org.apache.hadoop.mapred.InputFormat<K,V> getHadoopInputFormat() {
- return mapredInputFormat;
- }
-
- public void setHadoopInputFormat(org.apache.hadoop.mapred.InputFormat<K,V> mapredInputFormat) {
- this.mapredInputFormat = mapredInputFormat;
- }
-
- public JobConf getJobConf() {
- return jobConf;
- }
-
- // --------------------------------------------------------------------------------------------
- // InputFormat
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void configure(Configuration parameters) {
- // nothing to do
- }
-
- @Override
- public BaseStatistics getStatistics(BaseStatistics cachedStats) throws IOException {
- // only gather base statistics for FileInputFormats
- if(!(mapredInputFormat instanceof FileInputFormat)) {
- return null;
- }
-
- final FileBaseStatistics cachedFileStats = (cachedStats != null && cachedStats instanceof FileBaseStatistics) ?
- (FileBaseStatistics) cachedStats : null;
-
- try {
- final org.apache.hadoop.fs.Path[] paths = FileInputFormat.getInputPaths(this.jobConf);
-
- return getFileStats(cachedFileStats, paths, new ArrayList<FileStatus>(1));
- } catch (IOException ioex) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Could not determine statistics due to an io error: "
- + ioex.getMessage());
- }
- } catch (Throwable t) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Unexpected problem while getting the file statistics: "
- + t.getMessage(), t);
- }
- }
-
- // no statistics available
- return null;
- }
-
- @Override
- public HadoopInputSplit[] createInputSplits(int minNumSplits)
- throws IOException {
- org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
- HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
- for(int i=0;i<splitArray.length;i++){
- hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
- }
- return hiSplit;
- }
-
- @Override
- public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
- return new LocatableInputSplitAssigner(inputSplits);
- }
-
- @Override
- public void open(HadoopInputSplit split) throws IOException {
- this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
- if (this.recordReader instanceof Configurable) {
- ((Configurable) this.recordReader).setConf(jobConf);
- }
- key = this.recordReader.createKey();
- value = this.recordReader.createValue();
- this.fetched = false;
- }
-
- @Override
- public boolean reachedEnd() throws IOException {
- if(!fetched) {
- fetchNext();
- }
- return !hasNext;
- }
-
- private void fetchNext() throws IOException {
- hasNext = this.recordReader.next(key, value);
- fetched = true;
- }
-
- @Override
- public Tuple2<K, V> nextRecord(Tuple2<K, V> record) throws IOException {
- if(!fetched) {
- fetchNext();
- }
- if(!hasNext) {
- return null;
- }
- record.f0 = key;
- record.f1 = value;
- fetched = false;
- return record;
- }
-
- @Override
- public void close() throws IOException {
- this.recordReader.close();
- }
-
- // --------------------------------------------------------------------------------------------
- // Helper methods
- // --------------------------------------------------------------------------------------------
-
- private FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, org.apache.hadoop.fs.Path[] hadoopFilePaths,
- ArrayList<FileStatus> files) throws IOException {
-
- long latestModTime = 0L;
-
- // get the file info and check whether the cached statistics are still valid.
- for(org.apache.hadoop.fs.Path hadoopPath : hadoopFilePaths) {
-
- final Path filePath = new Path(hadoopPath.toUri());
- final FileSystem fs = FileSystem.get(filePath.toUri());
-
- final FileStatus file = fs.getFileStatus(filePath);
- latestModTime = Math.max(latestModTime, file.getModificationTime());
-
- // enumerate all files and check their modification time stamp.
- if (file.isDir()) {
- FileStatus[] fss = fs.listStatus(filePath);
- files.ensureCapacity(files.size() + fss.length);
-
- for (FileStatus s : fss) {
- if (!s.isDir()) {
- files.add(s);
- latestModTime = Math.max(s.getModificationTime(), latestModTime);
- }
- }
- } else {
- files.add(file);
- }
- }
-
- // check whether the cached statistics are still valid, if we have any
- if (cachedStats != null && latestModTime <= cachedStats.getLastModificationTime()) {
- return cachedStats;
- }
-
- // calculate the whole length
- long len = 0;
- for (FileStatus s : files) {
- len += s.getLen();
- }
-
- // sanity check
- if (len <= 0) {
- len = BaseStatistics.SIZE_UNKNOWN;
- }
-
- return new FileBaseStatistics(latestModTime, len, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
- }
-
- // --------------------------------------------------------------------------------------------
- // Custom serialization methods
- // --------------------------------------------------------------------------------------------
-
- private void writeObject(ObjectOutputStream out) throws IOException {
- out.writeUTF(mapredInputFormat.getClass().getName());
- out.writeUTF(keyClass.getName());
- out.writeUTF(valueClass.getName());
- jobConf.write(out);
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
- String hadoopInputFormatClassName = in.readUTF();
- String keyClassName = in.readUTF();
- String valueClassName = in.readUTF();
- if(jobConf == null) {
- jobConf = new JobConf();
- }
- jobConf.readFields(in);
- try {
- this.mapredInputFormat = (org.apache.hadoop.mapred.InputFormat<K,V>) Class.forName(hadoopInputFormatClassName, true, Thread.currentThread().getContextClassLoader()).newInstance();
- } catch (Exception e) {
- throw new RuntimeException("Unable to instantiate the hadoop input format", e);
- }
- try {
- this.keyClass = (Class<K>) Class.forName(keyClassName, true, Thread.currentThread().getContextClassLoader());
- } catch (Exception e) {
- throw new RuntimeException("Unable to find key class.", e);
- }
- try {
- this.valueClass = (Class<V>) Class.forName(valueClassName, true, Thread.currentThread().getContextClassLoader());
- } catch (Exception e) {
- throw new RuntimeException("Unable to find value class.", e);
- }
- ReflectionUtils.setConf(mapredInputFormat, jobConf);
- }
-
- // --------------------------------------------------------------------------------------------
- // ResultTypeQueryable
- // --------------------------------------------------------------------------------------------
-
- @Override
- public TypeInformation<Tuple2<K,V>> getProducedType() {
- return new TupleTypeInfo<Tuple2<K,V>>(TypeExtractor.createTypeInfo(keyClass), TypeExtractor.createTypeInfo(valueClass));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopMapFunction.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopMapFunction.java
deleted file mode 100644
index dfe0067..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopMapFunction.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapred;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopOutputCollector;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.Reporter;
-
-/**
- * This wrapper maps a Hadoop Mapper (mapred API) to a Flink FlatMapFunction.
- */
-@SuppressWarnings("rawtypes")
-public final class HadoopMapFunction<KEYIN, VALUEIN, KEYOUT, VALUEOUT>
- extends RichFlatMapFunction<Tuple2<KEYIN,VALUEIN>, Tuple2<KEYOUT,VALUEOUT>>
- implements ResultTypeQueryable<Tuple2<KEYOUT,VALUEOUT>>, Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private transient Mapper<KEYIN,VALUEIN,KEYOUT,VALUEOUT> mapper;
- private transient JobConf jobConf;
-
- private transient HadoopOutputCollector<KEYOUT,VALUEOUT> outputCollector;
- private transient Reporter reporter;
-
- /**
- * Maps a Hadoop Mapper (mapred API) to a Flink FlatMapFunction.
- *
- * @param hadoopMapper The Hadoop Mapper to wrap.
- */
- public HadoopMapFunction(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopMapper) {
- this(hadoopMapper, new JobConf());
- }
-
- /**
- * Maps a Hadoop Mapper (mapred API) to a Flink FlatMapFunction.
- * The Hadoop Mapper is configured with the provided JobConf.
- *
- * @param hadoopMapper The Hadoop Mapper to wrap.
- * @param conf The JobConf that is used to configure the Hadoop Mapper.
- */
- public HadoopMapFunction(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopMapper, JobConf conf) {
- if(hadoopMapper == null) {
- throw new NullPointerException("Mapper may not be null.");
- }
- if(conf == null) {
- throw new NullPointerException("JobConf may not be null.");
- }
-
- this.mapper = hadoopMapper;
- this.jobConf = conf;
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- this.mapper.configure(jobConf);
-
- this.reporter = new HadoopDummyReporter();
- this.outputCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
- }
-
- @Override
- public void flatMap(final Tuple2<KEYIN,VALUEIN> value, final Collector<Tuple2<KEYOUT,VALUEOUT>> out)
- throws Exception {
- outputCollector.setFlinkCollector(out);
- mapper.map(value.f0, value.f1, outputCollector, reporter);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public TypeInformation<Tuple2<KEYOUT,VALUEOUT>> getProducedType() {
- Class<KEYOUT> outKeyClass = (Class<KEYOUT>) TypeExtractor.getParameterType(Mapper.class, mapper.getClass(), 2);
- Class<VALUEOUT> outValClass = (Class<VALUEOUT>)TypeExtractor.getParameterType(Mapper.class, mapper.getClass(), 3);
-
- final TypeInformation<KEYOUT> keyTypeInfo = TypeExtractor.getForClass((Class<KEYOUT>) outKeyClass);
- final TypeInformation<VALUEOUT> valueTypleInfo = TypeExtractor.getForClass((Class<VALUEOUT>) outValClass);
- return new TupleTypeInfo<Tuple2<KEYOUT,VALUEOUT>>(keyTypeInfo, valueTypleInfo);
- }
-
- /**
- * Custom serialization methods.
- * @see http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html
- */
- private void writeObject(final ObjectOutputStream out) throws IOException {
- out.writeObject(mapper.getClass());
- jobConf.write(out);
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
- Class<Mapper<KEYIN,VALUEIN,KEYOUT,VALUEOUT>> mapperClass =
- (Class<Mapper<KEYIN,VALUEIN,KEYOUT,VALUEOUT>>)in.readObject();
- mapper = InstantiationUtil.instantiate(mapperClass);
-
- jobConf = new JobConf();
- jobConf.readFields(in);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopOutputFormat.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopOutputFormat.java
deleted file mode 100644
index f3abfcd..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopOutputFormat.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
-import org.apache.flink.api.common.io.FinalizeOnMaster;
-import org.apache.flink.api.common.io.OutputFormat;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.hadoopcompatibility.mapred.utils.HadoopUtils;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyProgressable;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
-import org.apache.hadoop.mapred.FileOutputCommitter;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
-import org.apache.hadoop.mapred.JobID;
-import org.apache.hadoop.mapred.RecordWriter;
-import org.apache.hadoop.mapred.TaskAttemptContext;
-import org.apache.hadoop.mapred.TaskAttemptID;
-import org.apache.hadoop.util.ReflectionUtils;
-
-
-public class HadoopOutputFormat<K,V> implements OutputFormat<Tuple2<K,V>>, FinalizeOnMaster {
-
- private static final long serialVersionUID = 1L;
-
- private JobConf jobConf;
- private org.apache.hadoop.mapred.OutputFormat<K,V> mapredOutputFormat;
- private transient RecordWriter<K,V> recordWriter;
- private transient FileOutputCommitter fileOutputCommitter;
- private transient TaskAttemptContext context;
- private transient JobContext jobContext;
-
- public HadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K,V> mapredOutputFormat, JobConf job) {
- super();
- this.mapredOutputFormat = mapredOutputFormat;
- HadoopUtils.mergeHadoopConf(job);
- this.jobConf = job;
- }
-
- public void setJobConf(JobConf job) {
- this.jobConf = job;
- }
-
- public JobConf getJobConf() {
- return jobConf;
- }
-
- public org.apache.hadoop.mapred.OutputFormat<K,V> getHadoopOutputFormat() {
- return mapredOutputFormat;
- }
-
- public void setHadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K,V> mapredOutputFormat) {
- this.mapredOutputFormat = mapredOutputFormat;
- }
-
- // --------------------------------------------------------------------------------------------
- // OutputFormat
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void configure(Configuration parameters) {
- // nothing to do
- }
-
- /**
- * create the temporary output file for hadoop RecordWriter.
- * @param taskNumber The number of the parallel instance.
- * @param numTasks The number of parallel tasks.
- * @throws IOException
- */
- @Override
- public void open(int taskNumber, int numTasks) throws IOException {
- if (Integer.toString(taskNumber + 1).length() > 6) {
- throw new IOException("Task id too large.");
- }
-
- TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_"
- + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s"," ").replace(" ", "0")
- + Integer.toString(taskNumber + 1)
- + "_0");
-
- this.jobConf.set("mapred.task.id", taskAttemptID.toString());
- this.jobConf.setInt("mapred.task.partition", taskNumber + 1);
- // for hadoop 2.2
- this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString());
- this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1);
-
- try {
- this.context = HadoopUtils.instantiateTaskAttemptContext(this.jobConf, taskAttemptID);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- this.fileOutputCommitter = new FileOutputCommitter();
-
- try {
- this.jobContext = HadoopUtils.instantiateJobContext(this.jobConf, new JobID());
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- this.fileOutputCommitter.setupJob(jobContext);
-
- this.recordWriter = this.mapredOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable());
- }
-
- @Override
- public void writeRecord(Tuple2<K, V> record) throws IOException {
- this.recordWriter.write(record.f0, record.f1);
- }
-
- /**
- * commit the task by moving the output file out from the temporary directory.
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- this.recordWriter.close(new HadoopDummyReporter());
-
- if (this.fileOutputCommitter.needsTaskCommit(this.context)) {
- this.fileOutputCommitter.commitTask(this.context);
- }
- }
-
- @Override
- public void finalizeGlobal(int parallelism) throws IOException {
-
- try {
- JobContext jobContext = HadoopUtils.instantiateJobContext(this.jobConf, new JobID());
- FileOutputCommitter fileOutputCommitter = new FileOutputCommitter();
-
- // finalize HDFS output format
- fileOutputCommitter.commitJob(jobContext);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Custom serialization methods
- // --------------------------------------------------------------------------------------------
-
- private void writeObject(ObjectOutputStream out) throws IOException {
- out.writeUTF(mapredOutputFormat.getClass().getName());
- jobConf.write(out);
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
- String hadoopOutputFormatName = in.readUTF();
- if(jobConf == null) {
- jobConf = new JobConf();
- }
- jobConf.readFields(in);
- try {
- this.mapredOutputFormat = (org.apache.hadoop.mapred.OutputFormat<K,V>) Class.forName(hadoopOutputFormatName, true, Thread.currentThread().getContextClassLoader()).newInstance();
- } catch (Exception e) {
- throw new RuntimeException("Unable to instantiate the hadoop output format", e);
- }
- ReflectionUtils.setConf(mapredOutputFormat, jobConf);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceCombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceCombineFunction.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceCombineFunction.java
deleted file mode 100644
index aa9f048..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceCombineFunction.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapred;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopOutputCollector;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopTupleUnwrappingIterator;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-/**
- * This wrapper maps a Hadoop Reducer and Combiner (mapred API) to a combinable Flink GroupReduceFunction.
- */
-@SuppressWarnings("rawtypes")
-@org.apache.flink.api.common.functions.RichGroupReduceFunction.Combinable
-public final class HadoopReduceCombineFunction<KEYIN, VALUEIN, KEYOUT, VALUEOUT>
- extends RichGroupReduceFunction<Tuple2<KEYIN,VALUEIN>,Tuple2<KEYOUT,VALUEOUT>>
- implements ResultTypeQueryable<Tuple2<KEYOUT,VALUEOUT>>, Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private transient Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT> reducer;
- private transient Reducer<KEYIN,VALUEIN,KEYIN,VALUEIN> combiner;
- private transient JobConf jobConf;
-
- private transient HadoopTupleUnwrappingIterator<KEYIN, VALUEIN> valueIterator;
- private transient HadoopOutputCollector<KEYOUT,VALUEOUT> reduceCollector;
- private transient HadoopOutputCollector<KEYIN,VALUEIN> combineCollector;
- private transient Reporter reporter;
-
- /**
- * Maps two Hadoop Reducer (mapred API) to a combinable Flink GroupReduceFunction.
- *
- * @param hadoopReducer The Hadoop Reducer that is mapped to a GroupReduceFunction.
- * @param hadoopCombiner The Hadoop Reducer that is mapped to the combiner function.
- */
- public HadoopReduceCombineFunction(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopReducer,
- Reducer<KEYIN,VALUEIN,KEYIN,VALUEIN> hadoopCombiner) {
- this(hadoopReducer, hadoopCombiner, new JobConf());
- }
-
- /**
- * Maps two Hadoop Reducer (mapred API) to a combinable Flink GroupReduceFunction.
- *
- * @param hadoopReducer The Hadoop Reducer that is mapped to a GroupReduceFunction.
- * @param hadoopCombiner The Hadoop Reducer that is mapped to the combiner function.
- * @param conf The JobConf that is used to configure both Hadoop Reducers.
- */
- public HadoopReduceCombineFunction(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopReducer,
- Reducer<KEYIN,VALUEIN,KEYIN,VALUEIN> hadoopCombiner, JobConf conf) {
- if(hadoopReducer == null) {
- throw new NullPointerException("Reducer may not be null.");
- }
- if(hadoopCombiner == null) {
- throw new NullPointerException("Combiner may not be null.");
- }
- if(conf == null) {
- throw new NullPointerException("JobConf may not be null.");
- }
-
- this.reducer = hadoopReducer;
- this.combiner = hadoopCombiner;
- this.jobConf = conf;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- this.reducer.configure(jobConf);
- this.combiner.configure(jobConf);
-
- this.reporter = new HadoopDummyReporter();
- Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
- this.valueIterator = new HadoopTupleUnwrappingIterator<KEYIN, VALUEIN>(inKeyClass);
- this.combineCollector = new HadoopOutputCollector<KEYIN, VALUEIN>();
- this.reduceCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
- }
-
- @Override
- public void reduce(final Iterable<Tuple2<KEYIN,VALUEIN>> values, final Collector<Tuple2<KEYOUT,VALUEOUT>> out)
- throws Exception {
- reduceCollector.setFlinkCollector(out);
- valueIterator.set(values.iterator());
- reducer.reduce(valueIterator.getCurrentKey(), valueIterator, reduceCollector, reporter);
- }
-
- @Override
- public void combine(final Iterable<Tuple2<KEYIN,VALUEIN>> values, final Collector<Tuple2<KEYIN,VALUEIN>> out) throws Exception {
- combineCollector.setFlinkCollector(out);
- valueIterator.set(values.iterator());
- combiner.reduce(valueIterator.getCurrentKey(), valueIterator, combineCollector, reporter);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public TypeInformation<Tuple2<KEYOUT,VALUEOUT>> getProducedType() {
- Class<KEYOUT> outKeyClass = (Class<KEYOUT>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 2);
- Class<VALUEOUT> outValClass = (Class<VALUEOUT>)TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 3);
-
- final TypeInformation<KEYOUT> keyTypeInfo = TypeExtractor.getForClass((Class<KEYOUT>) outKeyClass);
- final TypeInformation<VALUEOUT> valueTypleInfo = TypeExtractor.getForClass((Class<VALUEOUT>) outValClass);
- return new TupleTypeInfo<Tuple2<KEYOUT,VALUEOUT>>(keyTypeInfo, valueTypleInfo);
- }
-
- /**
- * Custom serialization methods.
- * @see http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html
- */
- private void writeObject(final ObjectOutputStream out) throws IOException {
-
- out.writeObject(reducer.getClass());
- out.writeObject(combiner.getClass());
- jobConf.write(out);
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
-
- Class<Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT>> reducerClass =
- (Class<Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT>>)in.readObject();
- reducer = InstantiationUtil.instantiate(reducerClass);
-
- Class<Reducer<KEYIN,VALUEIN,KEYIN,VALUEIN>> combinerClass =
- (Class<Reducer<KEYIN,VALUEIN,KEYIN,VALUEIN>>)in.readObject();
- combiner = InstantiationUtil.instantiate(combinerClass);
-
- jobConf = new JobConf();
- jobConf.readFields(in);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceFunction.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceFunction.java
deleted file mode 100644
index d9797c3..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/HadoopReduceFunction.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapred;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopOutputCollector;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopTupleUnwrappingIterator;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-/**
- * This wrapper maps a Hadoop Reducer (mapred API) to a non-combinable Flink GroupReduceFunction.
- */
-@SuppressWarnings("rawtypes")
-public final class HadoopReduceFunction<KEYIN, VALUEIN, KEYOUT, VALUEOUT>
- extends RichGroupReduceFunction<Tuple2<KEYIN,VALUEIN>,Tuple2<KEYOUT,VALUEOUT>>
- implements ResultTypeQueryable<Tuple2<KEYOUT,VALUEOUT>>, Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private transient Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT> reducer;
- private transient JobConf jobConf;
-
- private transient HadoopTupleUnwrappingIterator<KEYIN, VALUEIN> valueIterator;
- private transient HadoopOutputCollector<KEYOUT,VALUEOUT> reduceCollector;
- private transient Reporter reporter;
-
- /**
- * Maps a Hadoop Reducer (mapred API) to a non-combinable Flink GroupReduceFunction.
- *
- * @param hadoopReducer The Hadoop Reducer to wrap.
- */
- public HadoopReduceFunction(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopReducer) {
- this(hadoopReducer, new JobConf());
- }
-
- /**
- * Maps a Hadoop Reducer (mapred API) to a non-combinable Flink GroupReduceFunction.
- *
- * @param hadoopReducer The Hadoop Reducer to wrap.
- * @param conf The JobConf that is used to configure the Hadoop Reducer.
- */
- public HadoopReduceFunction(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> hadoopReducer, JobConf conf) {
- if(hadoopReducer == null) {
- throw new NullPointerException("Reducer may not be null.");
- }
- if(conf == null) {
- throw new NullPointerException("JobConf may not be null.");
- }
-
- this.reducer = hadoopReducer;
- this.jobConf = conf;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- this.reducer.configure(jobConf);
-
- this.reporter = new HadoopDummyReporter();
- this.reduceCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
- Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
- this.valueIterator = new HadoopTupleUnwrappingIterator<KEYIN, VALUEIN>(inKeyClass);
- }
-
- @Override
- public void reduce(final Iterable<Tuple2<KEYIN,VALUEIN>> values, final Collector<Tuple2<KEYOUT,VALUEOUT>> out)
- throws Exception {
-
- reduceCollector.setFlinkCollector(out);
- valueIterator.set(values.iterator());
- reducer.reduce(valueIterator.getCurrentKey(), valueIterator, reduceCollector, reporter);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public TypeInformation<Tuple2<KEYOUT,VALUEOUT>> getProducedType() {
- Class<KEYOUT> outKeyClass = (Class<KEYOUT>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 2);
- Class<VALUEOUT> outValClass = (Class<VALUEOUT>)TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 3);
-
- final TypeInformation<KEYOUT> keyTypeInfo = TypeExtractor.getForClass((Class<KEYOUT>) outKeyClass);
- final TypeInformation<VALUEOUT> valueTypleInfo = TypeExtractor.getForClass((Class<VALUEOUT>) outValClass);
- return new TupleTypeInfo<Tuple2<KEYOUT,VALUEOUT>>(keyTypeInfo, valueTypleInfo);
- }
-
- /**
- * Custom serialization methods.
- * @see http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html
- */
- private void writeObject(final ObjectOutputStream out) throws IOException {
-
- out.writeObject(reducer.getClass());
- jobConf.write(out);
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
-
- Class<Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT>> reducerClass =
- (Class<Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT>>)in.readObject();
- reducer = InstantiationUtil.instantiate(reducerClass);
-
- jobConf = new JobConf();
- jobConf.readFields(in);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/HadoopMapredCompatWordCount.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/HadoopMapredCompatWordCount.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/HadoopMapredCompatWordCount.java
deleted file mode 100644
index de20fab..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/HadoopMapredCompatWordCount.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapred.example;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.hadoopcompatibility.mapred.HadoopInputFormat;
-import org.apache.flink.hadoopcompatibility.mapred.HadoopMapFunction;
-import org.apache.flink.hadoopcompatibility.mapred.HadoopOutputFormat;
-import org.apache.flink.hadoopcompatibility.mapred.HadoopReduceCombineFunction;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.TextInputFormat;
-import org.apache.hadoop.mapred.TextOutputFormat;
-
-
-
-/**
- * Implements a word count which takes the input file and counts the number of
- * occurrences of each word in the file and writes the result back to disk.
- *
- * This example shows how to use Hadoop Input Formats, how to convert Hadoop Writables to
- * common Java types for better usage in a Flink job and how to use Hadoop Output Formats.
- */
-public class HadoopMapredCompatWordCount {
-
- public static void main(String[] args) throws Exception {
- if (args.length < 2) {
- System.err.println("Usage: WordCount <input path> <result path>");
- return;
- }
-
- final String inputPath = args[0];
- final String outputPath = args[1];
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- // Set up the Hadoop Input Format
- HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, new JobConf());
- TextInputFormat.addInputPath(hadoopInputFormat.getJobConf(), new Path(inputPath));
-
- // Create a Flink job with it
- DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);
-
- DataSet<Tuple2<Text, LongWritable>> words =
- text.flatMap(new HadoopMapFunction<LongWritable, Text, Text, LongWritable>(new Tokenizer()))
- .groupBy(0).reduceGroup(new HadoopReduceCombineFunction<Text, LongWritable, Text, LongWritable>(new Counter(), new Counter()));
-
- // Set up Hadoop Output Format
- HadoopOutputFormat<Text, LongWritable> hadoopOutputFormat =
- new HadoopOutputFormat<Text, LongWritable>(new TextOutputFormat<Text, LongWritable>(), new JobConf());
- hadoopOutputFormat.getJobConf().set("mapred.textoutputformat.separator", " ");
- TextOutputFormat.setOutputPath(hadoopOutputFormat.getJobConf(), new Path(outputPath));
-
- // Output & Execute
- words.output(hadoopOutputFormat).setParallelism(1);
- env.execute("Hadoop Compat WordCount");
- }
-
-
- public static final class Tokenizer implements Mapper<LongWritable, Text, Text, LongWritable> {
-
- @Override
- public void map(LongWritable k, Text v, OutputCollector<Text, LongWritable> out, Reporter rep)
- throws IOException {
- // normalize and split the line
- String line = v.toString();
- String[] tokens = line.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Text(token), new LongWritable(1l));
- }
- }
- }
-
- @Override
- public void configure(JobConf arg0) { }
-
- @Override
- public void close() throws IOException { }
-
- }
-
- public static final class Counter implements Reducer<Text, LongWritable, Text, LongWritable> {
-
- @Override
- public void reduce(Text k, Iterator<LongWritable> vs, OutputCollector<Text, LongWritable> out, Reporter rep)
- throws IOException {
-
- long cnt = 0;
- while(vs.hasNext()) {
- cnt += vs.next().get();
- }
- out.collect(k, new LongWritable(cnt));
-
- }
-
- @Override
- public void configure(JobConf arg0) { }
-
- @Override
- public void close() throws IOException { }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
deleted file mode 100644
index ff28a59..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.java.record.operators.GenericDataSink;
-import org.apache.flink.compiler.contextcheck.Validatable;
-import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.DefaultFlinkTypeConverter;
-import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.FlinkTypeConverter;
-import org.apache.flink.types.Record;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputFormat;
-
-/**
- * The HadoopDataSink is a generic wrapper for all Hadoop OutputFormats.
- *
- * Example usage:
- * <pre>
- * HadoopDataSink out = new HadoopDataSink(new org.apache.hadoop.mapred.TextOutputFormat<Text, IntWritable>(), new JobConf(), "Hadoop TextOutputFormat",reducer, Text.class,IntWritable.class);
- * org.apache.hadoop.mapred.TextOutputFormat.setOutputPath(out.getJobConf(), new Path(output));
- * </pre>
- *
- * Note that it is possible to provide custom data type converter.
- *
- * The HadoopDataSink provides a default converter: {@link org.apache.flink.hadoopcompatibility.mapred.record.datatypes.DefaultFlinkTypeConverter}
- **/
-public class HadoopDataSink<K,V> extends GenericDataSink implements Validatable {
-
- private static String DEFAULT_NAME = "<Unnamed Hadoop Data Sink>";
-
- private JobConf jobConf;
-
- public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, Operator<Record> input, FlinkTypeConverter<K,V> conv, Class<K> keyClass, Class<V> valueClass) {
- this(hadoopFormat, jobConf, name, Collections.<Operator<Record>>singletonList(input), conv, keyClass, valueClass);
- }
-
- public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, Operator<Record> input, Class<K> keyClass, Class<V> valueClass) {
- this(hadoopFormat, jobConf, name, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
- }
-
- public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, Operator<Record> input, Class<K> keyClass, Class<V> valueClass) {
- this(hadoopFormat, jobConf, DEFAULT_NAME, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
- }
-
- public HadoopDataSink(OutputFormat<K,V> hadoopFormat, Operator<Record> input, Class<K> keyClass, Class<V> valueClass) {
- this(hadoopFormat, new JobConf(), DEFAULT_NAME, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
- }
-
-
-
- @SuppressWarnings("deprecation")
- public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, List<Operator<Record>> input, FlinkTypeConverter<K,V> conv, Class<K> keyClass, Class<V> valueClass) {
- super(new HadoopRecordOutputFormat<K,V>(hadoopFormat, jobConf, conv),input, name);
-
- if (hadoopFormat == null || jobConf == null) {
- throw new NullPointerException();
- }
-
- this.name = name;
- this.jobConf = jobConf;
- jobConf.setOutputKeyClass(keyClass);
- jobConf.setOutputValueClass(valueClass);
- }
-
- public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, List<Operator<Record>> input, Class<K> keyClass, Class<V> valueClass) {
- this(hadoopFormat, jobConf, name, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
- }
-
- public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, List<Operator<Record>> input, Class<K> keyClass, Class<V> valueClass) {
- this(hadoopFormat, jobConf, DEFAULT_NAME, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
- }
-
- public HadoopDataSink(OutputFormat<K,V> hadoopFormat, List<Operator<Record>> input, Class<K> keyClass, Class<V> valueClass) {
- this(hadoopFormat, new JobConf(), DEFAULT_NAME, input, new DefaultFlinkTypeConverter<K, V>(keyClass, valueClass), keyClass, valueClass);
- }
-
- public JobConf getJobConf() {
- return this.jobConf;
- }
-
- @Override
- public void check() {
- // see for more details https://github.com/stratosphere/stratosphere/pull/531
- if (FileOutputFormat.getOutputPath(jobConf) == null) {
- throw new NullPointerException("The HadoopDataSink currently expects a correct outputPath.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSource.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSource.java
deleted file mode 100644
index 508f069..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSource.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.hadoopcompatibility.mapred.record;
-
-
-import org.apache.flink.api.java.record.operators.GenericDataSource;
-import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.DefaultHadoopTypeConverter;
-import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.HadoopTypeConverter;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.JobConf;
-
-/**
- * The HadoopDataSource is a generic wrapper for all Hadoop InputFormats.
- *
- * Example usage:
- * <pre>
- * HadoopDataSource source = new HadoopDataSource(new org.apache.hadoop.mapred.TextInputFormat(), new JobConf(), "Input Lines");
- * org.apache.hadoop.mapred.TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
- * </pre>
- *
- * Note that it is possible to provide custom data type converter.
- *
- * The HadoopDataSource provides two different standard converters:
- * * WritableWrapperConverter: Converts Hadoop Types to a record that contains a WritableComparableWrapper (key) and a WritableWrapper
- * * DefaultHadoopTypeConverter: Converts the standard hadoop types (longWritable, Text) to Flinks's {@link org.apache.flink.types.Value} types.
- */
-public class HadoopDataSource<K,V> extends GenericDataSource<HadoopRecordInputFormat<K,V>> {
-
- private static String DEFAULT_NAME = "<Unnamed Hadoop Data Source>";
-
- private JobConf jobConf;
-
- /**
- *
- * @param hadoopFormat Implementation of a Hadoop input format
- * @param jobConf JobConf object (Hadoop)
- * @param name Name of the DataSource
- * @param conv Definition of a custom type converter {@link DefaultHadoopTypeConverter}.
- */
- public HadoopDataSource(InputFormat<K,V> hadoopFormat, JobConf jobConf, String name, HadoopTypeConverter<K,V> conv) {
- super(new HadoopRecordInputFormat<K,V>(hadoopFormat, jobConf, conv),name);
-
- if (hadoopFormat == null || jobConf == null || conv == null) {
- throw new NullPointerException();
- }
-
- this.name = name;
- this.jobConf = jobConf;
- }
-
- public HadoopDataSource(InputFormat<K,V> hadoopFormat, JobConf jobConf, String name) {
- this(hadoopFormat, jobConf, name, new DefaultHadoopTypeConverter<K,V>() );
- }
- public HadoopDataSource(InputFormat<K,V> hadoopFormat, JobConf jobConf) {
- this(hadoopFormat, jobConf, DEFAULT_NAME);
- }
-
- public HadoopDataSource(InputFormat<K,V> hadoopFormat) {
- this(hadoopFormat, new JobConf(), DEFAULT_NAME);
- }
-
- public JobConf getJobConf() {
- return this.jobConf;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordInputFormat.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordInputFormat.java
deleted file mode 100644
index 275fd4c..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordInputFormat.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
-import org.apache.flink.api.common.io.DefaultInputSplitAssigner;
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.io.statistics.BaseStatistics;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.io.InputSplitAssigner;
-import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.HadoopTypeConverter;
-import org.apache.flink.hadoopcompatibility.mapred.utils.HadoopUtils;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopInputSplit;
-import org.apache.flink.types.Record;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.util.ReflectionUtils;
-
-public class HadoopRecordInputFormat<K, V> implements InputFormat<Record, HadoopInputSplit> {
-
- private static final long serialVersionUID = 1L;
-
- public org.apache.hadoop.mapred.InputFormat<K, V> hadoopInputFormat;
- public HadoopTypeConverter<K,V> converter;
- private String hadoopInputFormatName;
- public JobConf jobConf;
- public transient K key;
- public transient V value;
- public RecordReader<K, V> recordReader;
- private boolean fetched = false;
- private boolean hasNext;
-
- public HadoopRecordInputFormat() {
- super();
- }
-
- public HadoopRecordInputFormat(org.apache.hadoop.mapred.InputFormat<K,V> hadoopInputFormat, JobConf job, HadoopTypeConverter<K,V> conv) {
- super();
- this.hadoopInputFormat = hadoopInputFormat;
- this.hadoopInputFormatName = hadoopInputFormat.getClass().getName();
- this.converter = conv;
- HadoopUtils.mergeHadoopConf(job);
- this.jobConf = job;
- }
-
- @Override
- public void configure(Configuration parameters) {
-
- }
-
- @Override
- public BaseStatistics getStatistics(BaseStatistics cachedStatistics) throws IOException {
- return null;
- }
-
- @Override
- public HadoopInputSplit[] createInputSplits(int minNumSplits)
- throws IOException {
- org.apache.hadoop.mapred.InputSplit[] splitArray = hadoopInputFormat.getSplits(jobConf, minNumSplits);
- HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
- for(int i=0;i<splitArray.length;i++){
- hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
- }
- return hiSplit;
- }
-
- @Override
- public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
- return new DefaultInputSplitAssigner(inputSplits);
- }
-
- @Override
- public void open(HadoopInputSplit split) throws IOException {
- this.recordReader = this.hadoopInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
- key = this.recordReader.createKey();
- value = this.recordReader.createValue();
- this.fetched = false;
- }
-
- private void fetchNext() throws IOException {
- hasNext = this.recordReader.next(key, value);
- fetched = true;
- }
-
- @Override
- public boolean reachedEnd() throws IOException {
- if(!fetched) {
- fetchNext();
- }
- return !hasNext;
- }
-
- @Override
- public Record nextRecord(Record record) throws IOException {
- if(!fetched) {
- fetchNext();
- }
- if(!hasNext) {
- return null;
- }
- converter.convert(record, key, value);
- fetched = false;
- return record;
- }
-
- @Override
- public void close() throws IOException {
- this.recordReader.close();
- }
-
- /**
- * Custom serialization methods.
- * @see "http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html"
- */
- private void writeObject(ObjectOutputStream out) throws IOException {
- out.writeUTF(hadoopInputFormatName);
- jobConf.write(out);
- out.writeObject(converter);
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
- hadoopInputFormatName = in.readUTF();
- if(jobConf == null) {
- jobConf = new JobConf();
- }
- jobConf.readFields(in);
- try {
- this.hadoopInputFormat = (org.apache.hadoop.mapred.InputFormat<K,V>) Class.forName(this.hadoopInputFormatName).newInstance();
- } catch (Exception e) {
- throw new RuntimeException("Unable to instantiate the hadoop input format", e);
- }
- ReflectionUtils.setConf(hadoopInputFormat, jobConf);
- converter = (HadoopTypeConverter<K,V>) in.readObject();
- }
-
- public void setJobConf(JobConf job) {
- this.jobConf = job;
- }
-
-
- public org.apache.hadoop.mapred.InputFormat<K,V> getHadoopInputFormat() {
- return hadoopInputFormat;
- }
-
- public void setHadoopInputFormat(org.apache.hadoop.mapred.InputFormat<K,V> hadoopInputFormat) {
- this.hadoopInputFormat = hadoopInputFormat;
- }
-
- public JobConf getJobConf() {
- return jobConf;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordOutputFormat.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordOutputFormat.java
deleted file mode 100644
index 74118a3..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopRecordOutputFormat.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
-import org.apache.flink.api.common.io.OutputFormat;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.HadoopFileOutputCommitter;
-import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.FlinkTypeConverter;
-import org.apache.flink.hadoopcompatibility.mapred.utils.HadoopUtils;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyProgressable;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter;
-import org.apache.flink.types.Record;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordWriter;
-import org.apache.hadoop.mapred.TaskAttemptID;
-import org.apache.hadoop.util.ReflectionUtils;
-
-
-public class HadoopRecordOutputFormat<K,V> implements OutputFormat<Record> {
-
- private static final long serialVersionUID = 1L;
-
- public JobConf jobConf;
-
- public org.apache.hadoop.mapred.OutputFormat<K,V> hadoopOutputFormat;
-
- private String hadoopOutputFormatName;
-
- public RecordWriter<K,V> recordWriter;
-
- public FlinkTypeConverter<K,V> converter;
-
- public HadoopFileOutputCommitter fileOutputCommitterWrapper;
-
- public HadoopRecordOutputFormat(org.apache.hadoop.mapred.OutputFormat<K,V> hadoopFormat, JobConf job, FlinkTypeConverter<K,V> conv) {
- super();
- this.hadoopOutputFormat = hadoopFormat;
- this.hadoopOutputFormatName = hadoopFormat.getClass().getName();
- this.converter = conv;
- this.fileOutputCommitterWrapper = new HadoopFileOutputCommitter();
- HadoopUtils.mergeHadoopConf(job);
- this.jobConf = job;
- }
-
- @Override
- public void configure(Configuration parameters) {
- }
-
- /**
- * create the temporary output file for hadoop RecordWriter.
- * @param taskNumber The number of the parallel instance.
- * @param numTasks The number of parallel tasks.
- * @throws IOException
- */
- @Override
- public void open(int taskNumber, int numTasks) throws IOException {
- this.fileOutputCommitterWrapper.setupJob(this.jobConf);
- if (Integer.toString(taskNumber + 1).length() <= 6) {
- this.jobConf.set("mapred.task.id", "attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s"," ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0");
- //compatible for hadoop 2.2.0, the temporary output directory is different from hadoop 1.2.1
- this.jobConf.set("mapreduce.task.output.dir", this.fileOutputCommitterWrapper.getTempTaskOutputPath(this.jobConf,TaskAttemptID.forName(this.jobConf.get("mapred.task.id"))).toString());
- } else {
- throw new IOException("task id too large");
- }
- this.recordWriter = this.hadoopOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable());
- }
-
-
- @Override
- public void writeRecord(Record record) throws IOException {
- K key = this.converter.convertKey(record);
- V value = this.converter.convertValue(record);
- this.recordWriter.write(key, value);
- }
-
- /**
- * commit the task by moving the output file out from the temporary directory.
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- this.recordWriter.close(new HadoopDummyReporter());
- if (this.fileOutputCommitterWrapper.needsTaskCommit(this.jobConf, TaskAttemptID.forName(this.jobConf.get("mapred.task.id")))) {
- this.fileOutputCommitterWrapper.commitTask(this.jobConf, TaskAttemptID.forName(this.jobConf.get("mapred.task.id")));
- }
- //TODO: commitjob when all the tasks are finished
- }
-
-
- /**
- * Custom serialization methods.
- * @see "http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html"
- */
- private void writeObject(ObjectOutputStream out) throws IOException {
- out.writeUTF(hadoopOutputFormatName);
- jobConf.write(out);
- out.writeObject(converter);
- out.writeObject(fileOutputCommitterWrapper);
- }
-
- @SuppressWarnings("unchecked")
- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
- hadoopOutputFormatName = in.readUTF();
- if(jobConf == null) {
- jobConf = new JobConf();
- }
- jobConf.readFields(in);
- try {
- this.hadoopOutputFormat = (org.apache.hadoop.mapred.OutputFormat<K,V>) Class.forName(this.hadoopOutputFormatName).newInstance();
- } catch (Exception e) {
- throw new RuntimeException("Unable to instantiate the hadoop output format", e);
- }
- ReflectionUtils.setConf(hadoopOutputFormat, jobConf);
- converter = (FlinkTypeConverter<K,V>) in.readObject();
- fileOutputCommitterWrapper = (HadoopFileOutputCommitter) in.readObject();
- }
-
-
- public void setJobConf(JobConf job) {
- this.jobConf = job;
- }
-
- public JobConf getJobConf() {
- return jobConf;
- }
-
- public org.apache.hadoop.mapred.OutputFormat<K,V> getHadoopOutputFormat() {
- return hadoopOutputFormat;
- }
-
- public void setHadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K,V> hadoopOutputFormat) {
- this.hadoopOutputFormat = hadoopOutputFormat;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultFlinkTypeConverter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultFlinkTypeConverter.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultFlinkTypeConverter.java
deleted file mode 100644
index 9d37988..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultFlinkTypeConverter.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
-
-import org.apache.flink.types.BooleanValue;
-import org.apache.flink.types.ByteValue;
-import org.apache.flink.types.DoubleValue;
-import org.apache.flink.types.FloatValue;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.StringValue;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.ByteWritable;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-
-/**
- * Convert Flink Record into the default hadoop writables.
- */
-public class DefaultFlinkTypeConverter<K,V> implements FlinkTypeConverter<K,V> {
- private static final long serialVersionUID = 1L;
-
- private Class<K> keyClass;
- private Class<V> valueClass;
-
- public DefaultFlinkTypeConverter(Class<K> keyClass, Class<V> valueClass) {
- this.keyClass= keyClass;
- this.valueClass = valueClass;
- }
- @Override
- public K convertKey(Record flinkRecord) {
- if(flinkRecord.getNumFields() > 0) {
- return convert(flinkRecord, 0, this.keyClass);
- } else {
- return null;
- }
- }
-
- @Override
- public V convertValue(Record flinkRecord) {
- if(flinkRecord.getNumFields() > 1) {
- return convert(flinkRecord, 1, this.valueClass);
- } else {
- return null;
- }
- }
-
- @SuppressWarnings("unchecked")
- private<T> T convert(Record flinkType, int pos, Class<T> hadoopType) {
- if(hadoopType == LongWritable.class ) {
- return (T) new LongWritable((flinkType.getField(pos, LongValue.class)).getValue());
- }
- if(hadoopType == org.apache.hadoop.io.Text.class) {
- return (T) new Text((flinkType.getField(pos, StringValue.class)).getValue());
- }
- if(hadoopType == org.apache.hadoop.io.IntWritable.class) {
- return (T) new IntWritable((flinkType.getField(pos, IntValue.class)).getValue());
- }
- if(hadoopType == org.apache.hadoop.io.FloatWritable.class) {
- return (T) new FloatWritable((flinkType.getField(pos, FloatValue.class)).getValue());
- }
- if(hadoopType == org.apache.hadoop.io.DoubleWritable.class) {
- return (T) new DoubleWritable((flinkType.getField(pos, DoubleValue.class)).getValue());
- }
- if(hadoopType == org.apache.hadoop.io.BooleanWritable.class) {
- return (T) new BooleanWritable((flinkType.getField(pos, BooleanValue.class)).getValue());
- }
- if(hadoopType == org.apache.hadoop.io.ByteWritable.class) {
- return (T) new ByteWritable((flinkType.getField(pos, ByteValue.class)).getValue());
- }
-
- throw new RuntimeException("Unable to convert Flink type ("+flinkType.getClass().getCanonicalName()+") to Hadoop.");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultHadoopTypeConverter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultHadoopTypeConverter.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultHadoopTypeConverter.java
deleted file mode 100644
index 6ed670a..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/DefaultHadoopTypeConverter.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
-
-import org.apache.flink.types.BooleanValue;
-import org.apache.flink.types.ByteValue;
-import org.apache.flink.types.DoubleValue;
-import org.apache.flink.types.FloatValue;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-import org.apache.flink.types.NullValue;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.StringValue;
-import org.apache.flink.types.Value;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.ByteWritable;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-
-
-/**
- * Converter for the default hadoop writables.
- * Key will be in field 0, Value in field 1 of a Record.
- */
-public class DefaultHadoopTypeConverter<K, V> implements HadoopTypeConverter<K, V> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void convert(Record flinkRecord, K hadoopKey, V hadoopValue) {
- flinkRecord.setField(0, convert(hadoopKey));
- flinkRecord.setField(1, convert(hadoopValue));
- }
-
- protected Value convert(Object hadoopType) {
- if(hadoopType instanceof org.apache.hadoop.io.LongWritable ) {
- return new LongValue(((LongWritable)hadoopType).get());
- }
- if(hadoopType instanceof org.apache.hadoop.io.Text) {
- return new StringValue(((Text)hadoopType).toString());
- }
- if(hadoopType instanceof org.apache.hadoop.io.IntWritable) {
- return new IntValue(((IntWritable)hadoopType).get());
- }
- if(hadoopType instanceof org.apache.hadoop.io.FloatWritable) {
- return new FloatValue(((FloatWritable)hadoopType).get());
- }
- if(hadoopType instanceof org.apache.hadoop.io.DoubleWritable) {
- return new DoubleValue(((DoubleWritable)hadoopType).get());
- }
- if(hadoopType instanceof org.apache.hadoop.io.BooleanWritable) {
- return new BooleanValue(((BooleanWritable)hadoopType).get());
- }
- if(hadoopType instanceof org.apache.hadoop.io.ByteWritable) {
- return new ByteValue(((ByteWritable)hadoopType).get());
- }
- if (hadoopType instanceof NullWritable) {
- return NullValue.getInstance();
- }
-
- throw new RuntimeException("Unable to convert Hadoop type ("+hadoopType.getClass().getCanonicalName()+") to a Flink data type.");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/FlinkTypeConverter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/FlinkTypeConverter.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/FlinkTypeConverter.java
deleted file mode 100644
index 3c14a86..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/FlinkTypeConverter.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
-
-import java.io.Serializable;
-
-import org.apache.flink.types.Record;
-
-/**
- * An interface describing a class that is able to
- * convert Flink's Record into Hadoop types model.
- *
- * The converter must be Serializable.
- *
- * Flink provides a DefaultFlinkTypeConverter. Custom implementations should
- * chain the type converters.
- */
-public interface FlinkTypeConverter<K,V> extends Serializable {
-
- /**
- * Convert a Flink type to a Hadoop type.
- */
- public K convertKey(Record record);
-
- public V convertValue(Record record);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopFileOutputCommitter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopFileOutputCommitter.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopFileOutputCommitter.java
deleted file mode 100644
index ce4955c..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopFileOutputCommitter.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.net.URI;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileOutputCommitter;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TaskAttemptID;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * Hadoop 1.2.1 {@link org.apache.hadoop.mapred.FileOutputCommitter} takes {@link org.apache.hadoop.mapred.JobContext}
- * as input parameter. However JobContext class is package private, and in Hadoop 2.2.0 it's public.
- * This class takes {@link org.apache.hadoop.mapred.JobConf} as input instead of JobContext in order to setup and commit tasks.
- */
-public class HadoopFileOutputCommitter extends FileOutputCommitter implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- static final String SUCCESSFUL_JOB_OUTPUT_DIR_MARKER =
- "mapreduce.fileoutputcommitter.marksuccessfuljobs";
-
- public void setupJob(JobConf conf) throws IOException {
- Path outputPath = FileOutputFormat.getOutputPath(conf);
- if (outputPath != null) {
- Path tmpDir = new Path(outputPath, FileOutputCommitter.TEMP_DIR_NAME);
- FileSystem fileSys = tmpDir.getFileSystem(conf);
- if (!fileSys.mkdirs(tmpDir)) {
- LOG.error("Mkdirs failed to create " + tmpDir.toString());
- }
- }
- }
-
- private static boolean getOutputDirMarking(JobConf conf) {
- return conf.getBoolean(SUCCESSFUL_JOB_OUTPUT_DIR_MARKER, true);
- }
-
- private void markSuccessfulOutputDir(JobConf conf)
- throws IOException {
- Path outputPath = FileOutputFormat.getOutputPath(conf);
- if (outputPath != null) {
- FileSystem fileSys = outputPath.getFileSystem(conf);
- // create a file in the folder to mark it
- if (fileSys.exists(outputPath)) {
- Path filePath = new Path(outputPath, SUCCEEDED_FILE_NAME);
- fileSys.create(filePath).close();
- }
- }
- }
-
- private Path getFinalPath(Path jobOutputDir, Path taskOutput,
- Path taskOutputPath) throws IOException {
- URI taskOutputUri = taskOutput.toUri();
- URI relativePath = taskOutputPath.toUri().relativize(taskOutputUri);
- if (taskOutputUri == relativePath) {//taskOutputPath is not a parent of taskOutput
- throw new IOException("Can not get the relative path: base = " +
- taskOutputPath + " child = " + taskOutput);
- }
- if (relativePath.getPath().length() > 0) {
- return new Path(jobOutputDir, relativePath.getPath());
- } else {
- return jobOutputDir;
- }
- }
- private void moveTaskOutputs(JobConf conf, TaskAttemptID taskAttemptID,
- FileSystem fs,
- Path jobOutputDir,
- Path taskOutput)
- throws IOException {
- if (fs.isFile(taskOutput)) {
- Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput,
- getTempTaskOutputPath(conf, taskAttemptID));
- if (!fs.rename(taskOutput, finalOutputPath)) {
- if (!fs.delete(finalOutputPath, true)) {
- throw new IOException("Failed to delete earlier output of task: " +
- taskAttemptID);
- }
- if (!fs.rename(taskOutput, finalOutputPath)) {
- throw new IOException("Failed to save output of task: " +
- taskAttemptID);
- }
- }
- LOG.debug("Moved " + taskOutput + " to " + finalOutputPath);
- } else if(fs.getFileStatus(taskOutput).isDir()) {
- FileStatus[] paths = fs.listStatus(taskOutput);
- Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput,
- getTempTaskOutputPath(conf, taskAttemptID));
- fs.mkdirs(finalOutputPath);
- if (paths != null) {
- for (FileStatus path : paths) {
- moveTaskOutputs(conf,taskAttemptID, fs, jobOutputDir, path.getPath());
- }
- }
- }
- }
-
- public void commitTask(JobConf conf, TaskAttemptID taskAttemptID)
- throws IOException {
- Path taskOutputPath = getTempTaskOutputPath(conf, taskAttemptID);
- if (taskOutputPath != null) {
- FileSystem fs = taskOutputPath.getFileSystem(conf);
- if (fs.exists(taskOutputPath)) {
- Path jobOutputPath = taskOutputPath.getParent().getParent();
- // Move the task outputs to their final place
- moveTaskOutputs(conf,taskAttemptID, fs, jobOutputPath, taskOutputPath);
- // Delete the temporary task-specific output directory
- if (!fs.delete(taskOutputPath, true)) {
- LOG.info("Failed to delete the temporary output" +
- " directory of task: " + taskAttemptID + " - " + taskOutputPath);
- }
- LOG.info("Saved output of task '" + taskAttemptID + "' to " +
- jobOutputPath);
- }
- }
- }
- public boolean needsTaskCommit(JobConf conf, TaskAttemptID taskAttemptID)
- throws IOException {
- try {
- Path taskOutputPath = getTempTaskOutputPath(conf, taskAttemptID);
- if (taskOutputPath != null) {
- // Get the file-system for the task output directory
- FileSystem fs = taskOutputPath.getFileSystem(conf);
- // since task output path is created on demand,
- // if it exists, task needs a commit
- if (fs.exists(taskOutputPath)) {
- return true;
- }
- }
- } catch (IOException ioe) {
- throw ioe;
- }
- return false;
- }
-
- public Path getTempTaskOutputPath(JobConf conf, TaskAttemptID taskAttemptID) {
- Path outputPath = FileOutputFormat.getOutputPath(conf);
- if (outputPath != null) {
- Path p = new Path(outputPath,
- (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR +
- "_" + taskAttemptID.toString()));
- try {
- FileSystem fs = p.getFileSystem(conf);
- return p.makeQualified(fs);
- } catch (IOException ie) {
- LOG.warn(StringUtils.stringifyException(ie));
- return p;
- }
- }
- return null;
- }
- public void cleanupJob(JobConf conf) throws IOException {
- // do the clean up of temporary directory
- Path outputPath = FileOutputFormat.getOutputPath(conf);
- if (outputPath != null) {
- Path tmpDir = new Path(outputPath, FileOutputCommitter.TEMP_DIR_NAME);
- FileSystem fileSys = tmpDir.getFileSystem(conf);
- if (fileSys.exists(tmpDir)) {
- fileSys.delete(tmpDir, true);
- }
- } else {
- LOG.warn("Output path is null in cleanup");
- }
- }
-
- public void commitJob(JobConf conf) throws IOException {
- cleanupJob(conf);
- if (getOutputDirMarking(conf)) {
- markSuccessfulOutputDir(conf);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopTypeConverter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopTypeConverter.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopTypeConverter.java
deleted file mode 100644
index 6bbf077..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/datatypes/HadoopTypeConverter.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.hadoopcompatibility.mapred.record.datatypes;
-
-import java.io.Serializable;
-
-import org.apache.flink.types.Record;
-
-
-/**
- * An interface describing a class that is able to
- * convert Hadoop types into Flink's Record model.
- *
- * The converter must be Serializable.
- *
- * Flink provides a DefaultHadoopTypeConverter. Custom implementations should
- * chain the type converters.
- */
-public interface HadoopTypeConverter<K, V> extends Serializable {
-
- /**
- * Convert a Hadoop type to a Flink type.
- */
- public void convert(Record record, K hadoopKey, V hadoopValue);
-}
[26/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java b/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java
deleted file mode 100644
index e1235b6..0000000
--- a/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.tachyon;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.ExecutionEnvironmentFactory;
-import org.apache.flink.api.java.LocalEnvironment;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.examples.java.wordcount.WordCount;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import tachyon.client.InStream;
-import tachyon.client.OutStream;
-import tachyon.client.ReadType;
-import tachyon.client.TachyonFS;
-import tachyon.client.TachyonFile;
-import tachyon.client.WriteType;
-import tachyon.master.LocalTachyonCluster;
-
-import java.io.File;
-import java.io.StringWriter;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-public class TachyonFileSystemWrapperTest {
- private static final long TACHYON_WORKER_CAPACITY = 1024 * 1024 * 32;
- private static final String TACHYON_TEST_IN_FILE_NAME = "tachyontest";
- private static final String TACHYON_TEST_OUT_FILE_NAME = "result";
- private static final Path HADOOP_CONFIG_PATH;
-
- static {
- URL resource = TachyonFileSystemWrapperTest.class.getResource("/tachyonHadoopConf.xml");
- File file = null;
- try {
- file = new File(resource.toURI());
- } catch (URISyntaxException e) {
- throw new RuntimeException("Unable to load req. res", e);
- }
- if(!file.exists()) {
- throw new RuntimeException("Unable to load required resource");
- }
- HADOOP_CONFIG_PATH = new Path(file.getAbsolutePath());
- }
-
- private LocalTachyonCluster cluster;
- private TachyonFS client;
- private String input;
- private String output;
-
- @Before
- public void startTachyon() {
- try {
- cluster = new LocalTachyonCluster(TACHYON_WORKER_CAPACITY);
- cluster.start();
- client = cluster.getClient();
- int id = client.createFile("/" + TACHYON_TEST_IN_FILE_NAME, 1024 * 32);
- Assert.assertNotEquals("Unable to create file", -1, id);
-
- TachyonFile testFile = client.getFile(id);
- Assert.assertNotNull(testFile);
-
-
- OutStream outStream = testFile.getOutStream(WriteType.MUST_CACHE);
- for(int i = 0; i < 10; i++) {
- outStream.write("Hello Tachyon\n".getBytes());
- }
- outStream.close();
- final String tachyonBase = "tachyon://" + cluster.getMasterHostname() + ":" + cluster.getMasterPort();
- input = tachyonBase + "/" + TACHYON_TEST_IN_FILE_NAME;
- output = tachyonBase + "/" + TACHYON_TEST_OUT_FILE_NAME;
-
- } catch(Exception e) {
- e.printStackTrace();
- Assert.fail("Test preparation failed with exception: "+e.getMessage());
- }
- }
-
- @After
- public void stopTachyon() {
- try {
- cluster.stop();
- } catch(Exception e) {
- e.printStackTrace();
- Assert.fail("Test teardown failed with exception: "+e.getMessage());
- }
- }
- // Verify that Hadoop's FileSystem can load the TFS (Tachyon File System)
- @Test
- public void testHadoopLoadability() {
- try {
- Path tPath = new Path(input);
- Configuration conf = new Configuration();
- conf.addResource(HADOOP_CONFIG_PATH);
- Assert.assertEquals("tachyon.hadoop.TFS", conf.get("fs.tachyon.impl", null));
- tPath.getFileSystem(conf);
- } catch(Exception e) {
- e.printStackTrace();
- Assert.fail("Test failed with exception: "+e.getMessage());
- }
- }
-
-
- @Test
- public void testTachyon() {
- try {
- org.apache.flink.configuration.Configuration addHDConfToFlinkConf = new org.apache.flink.configuration.Configuration();
- addHDConfToFlinkConf.setString(ConfigConstants.HDFS_DEFAULT_CONFIG, HADOOP_CONFIG_PATH.toString());
- GlobalConfiguration.includeConfiguration(addHDConfToFlinkConf);
-
- new DopOneTestEnvironment(); // initialize DOP one
-
- WordCount.main(new String[]{input, output});
-
- // verify result
- TachyonFile resultFile = client.getFile("/" + TACHYON_TEST_OUT_FILE_NAME);
- Assert.assertNotNull("Result file has not been created", resultFile);
- InStream inStream = resultFile.getInStream(ReadType.CACHE);
- Assert.assertNotNull("Result file has not been created", inStream);
- StringWriter writer = new StringWriter();
- IOUtils.copy(inStream, writer);
- String resultString = writer.toString();
-
- Assert.assertEquals("hello 10\n" +
- "tachyon 10\n", resultString);
-
- } catch(Exception e) {
- e.printStackTrace();
- Assert.fail("Test failed with exception: "+e.getMessage());
- }
- }
-
- // package visible
- static final class DopOneTestEnvironment extends LocalEnvironment {
- static {
- initializeContextEnvironment(new ExecutionEnvironmentFactory() {
- @Override
- public ExecutionEnvironment createExecutionEnvironment() {
- LocalEnvironment le = new LocalEnvironment();
- le.setDegreeOfParallelism(1);
- return le;
- }
- });
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-tachyon/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/flink-addons/flink-tachyon/src/test/resources/log4j.properties b/flink-addons/flink-tachyon/src/test/resources/log4j.properties
deleted file mode 100644
index 2c9a677..0000000
--- a/flink-addons/flink-tachyon/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# Tachyon's test-jar dependency adds a log4j.properties file to classpath.
-# Until the issue is resolved (see https://github.com/amplab/tachyon/pull/571)
-# we provide a log4j.properties file ourselves.
-
-log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-tachyon/src/test/resources/tachyonHadoopConf.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-tachyon/src/test/resources/tachyonHadoopConf.xml b/flink-addons/flink-tachyon/src/test/resources/tachyonHadoopConf.xml
deleted file mode 100644
index 0af8190..0000000
--- a/flink-addons/flink-tachyon/src/test/resources/tachyonHadoopConf.xml
+++ /dev/null
@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-
-<configuration>
- <property>
- <name>fs.tachyon.impl</name>
- <value>tachyon.hadoop.TFS</value>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/pom.xml b/flink-addons/pom.xml
deleted file mode 100644
index ea45cdb..0000000
--- a/flink-addons/pom.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-parent</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-addons</artifactId>
- <name>flink-addons</name>
- <packaging>pom</packaging>
-
- <modules>
- <module>flink-avro</module>
- <module>flink-jdbc</module>
- <module>flink-spargel</module>
- <module>flink-hadoop-compatibility</module>
- <module>flink-streaming</module>
- <module>flink-hbase</module>
- </modules>
-
- <!-- See main pom.xml for explanation of profiles -->
- <profiles>
- <profile>
- <id>hadoop-2</id>
- <activation>
- <property>
- <!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh -->
- <!--hadoop2--><name>!hadoop.profile</name>
- </property>
- </activation>
- <modules>
- <!-- Include the Flink-tachyon project only for HD2.
- The HDFS minicluster interfaces changed between the two versions.
- -->
- <module>flink-tachyon</module>
- </modules>
- </profile>
- </profiles>
-
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/pom.xml b/flink-staging/flink-avro/pom.xml
new file mode 100644
index 0000000..15bb801
--- /dev/null
+++ b/flink-staging/flink-avro/pom.xml
@@ -0,0 +1,182 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-staging</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-avro</artifactId>
+ <name>flink-avro</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <!-- version is derived from base module -->
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-test-dependency</id>
+ <phase>process-test-classes</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.flink.api.avro.testjar.AvroExternalJarProgram</mainClass>
+ </manifest>
+ </archive>
+ <finalName>maven</finalName>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/test/assembly/test-assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!--Remove the AvroExternalJarProgram code from the test-classes directory since it musn't be in the
+ classpath when running the tests to actually test whether the user code class loader
+ is properly used.-->
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>2.5</version><!--$NO-MVN-MAN-VER$-->
+ <executions>
+ <execution>
+ <id>remove-avroexternalprogram</id>
+ <phase>process-test-classes</phase>
+ <goals>
+ <goal>clean</goal>
+ </goals>
+ <configuration>
+ <excludeDefaultDirectories>true</excludeDefaultDirectories>
+ <filesets>
+ <fileset>
+ <directory>${project.build.testOutputDirectory}</directory>
+ <includes>
+ <include>**/testjar/*.class</include>
+ </includes>
+ </fileset>
+ </filesets>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <versionRange>[2.4,)</versionRange>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore/>
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-clean-plugin</artifactId>
+ <versionRange>[1,)</versionRange>
+ <goals>
+ <goal>clean</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore/>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+ <profiles>
+ <profile>
+ <!-- A bug with java6 is causing the javadoc generation
+ to fail because the test case contains junit?
+ See https://github.com/stratosphere/stratosphere/pull/405#issuecomment-32591978
+ for further links -->
+ <id>disable-javadocs-in-java6</id>
+ <activation>
+ <jdk>(,1.6]</jdk> <!-- disable javadocs for java6 or lower. -->
+ </activation>
+ <properties>
+ <maven.javadoc.skip>true</maven.javadoc.skip>
+ </properties>
+ </profile>
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java
new file mode 100644
index 0000000..59da4cb
--- /dev/null
+++ b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.avro;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.avro.io.Decoder;
+import org.apache.avro.util.Utf8;
+
+
+public class DataInputDecoder extends Decoder {
+
+ private final Utf8 stringDecoder = new Utf8();
+
+ private DataInput in;
+
+ public void setIn(DataInput in) {
+ this.in = in;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // primitives
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void readNull() {}
+
+
+ @Override
+ public boolean readBoolean() throws IOException {
+ return in.readBoolean();
+ }
+
+ @Override
+ public int readInt() throws IOException {
+ return in.readInt();
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ return in.readLong();
+ }
+
+ @Override
+ public float readFloat() throws IOException {
+ return in.readFloat();
+ }
+
+ @Override
+ public double readDouble() throws IOException {
+ return in.readDouble();
+ }
+
+ @Override
+ public int readEnum() throws IOException {
+ return readInt();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // bytes
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void readFixed(byte[] bytes, int start, int length) throws IOException {
+ in.readFully(bytes, start, length);
+ }
+
+ @Override
+ public ByteBuffer readBytes(ByteBuffer old) throws IOException {
+ int length = readInt();
+ ByteBuffer result;
+ if (old != null && length <= old.capacity() && old.hasArray()) {
+ result = old;
+ result.clear();
+ } else {
+ result = ByteBuffer.allocate(length);
+ }
+ in.readFully(result.array(), result.arrayOffset() + result.position(), length);
+ result.limit(length);
+ return result;
+ }
+
+
+ @Override
+ public void skipFixed(int length) throws IOException {
+ skipBytes(length);
+ }
+
+ @Override
+ public void skipBytes() throws IOException {
+ int num = readInt();
+ skipBytes(num);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // strings
+ // --------------------------------------------------------------------------------------------
+
+
+ @Override
+ public Utf8 readString(Utf8 old) throws IOException {
+ int length = readInt();
+ Utf8 result = (old != null ? old : new Utf8());
+ result.setByteLength(length);
+
+ if (length > 0) {
+ in.readFully(result.getBytes(), 0, length);
+ }
+
+ return result;
+ }
+
+ @Override
+ public String readString() throws IOException {
+ return readString(stringDecoder).toString();
+ }
+
+ @Override
+ public void skipString() throws IOException {
+ int len = readInt();
+ skipBytes(len);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // collection types
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public long readArrayStart() throws IOException {
+ return readVarLongCount(in);
+ }
+
+ @Override
+ public long arrayNext() throws IOException {
+ return readVarLongCount(in);
+ }
+
+ @Override
+ public long skipArray() throws IOException {
+ return readVarLongCount(in);
+ }
+
+ @Override
+ public long readMapStart() throws IOException {
+ return readVarLongCount(in);
+ }
+
+ @Override
+ public long mapNext() throws IOException {
+ return readVarLongCount(in);
+ }
+
+ @Override
+ public long skipMap() throws IOException {
+ return readVarLongCount(in);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // union
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public int readIndex() throws IOException {
+ return readInt();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // utils
+ // --------------------------------------------------------------------------------------------
+
+ private void skipBytes(int num) throws IOException {
+ while (num > 0) {
+ num -= in.skipBytes(num);
+ }
+ }
+
+ public static long readVarLongCount(DataInput in) throws IOException {
+ long value = in.readUnsignedByte();
+
+ if ((value & 0x80) == 0) {
+ return value;
+ }
+ else {
+ long curr;
+ int shift = 7;
+ value = value & 0x7f;
+ while (((curr = in.readUnsignedByte()) & 0x80) != 0){
+ value |= (curr & 0x7f) << shift;
+ shift += 7;
+ }
+ value |= curr << shift;
+ return value;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java
new file mode 100644
index 0000000..0102cc1
--- /dev/null
+++ b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.avro;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.avro.io.Encoder;
+import org.apache.avro.util.Utf8;
+
+
+public final class DataOutputEncoder extends Encoder implements java.io.Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private DataOutput out;
+
+
+ public void setOut(DataOutput out) {
+ this.out = out;
+ }
+
+
+ @Override
+ public void flush() throws IOException {}
+
+ // --------------------------------------------------------------------------------------------
+ // primitives
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void writeNull() {}
+
+
+ @Override
+ public void writeBoolean(boolean b) throws IOException {
+ out.writeBoolean(b);
+ }
+
+ @Override
+ public void writeInt(int n) throws IOException {
+ out.writeInt(n);
+ }
+
+ @Override
+ public void writeLong(long n) throws IOException {
+ out.writeLong(n);
+ }
+
+ @Override
+ public void writeFloat(float f) throws IOException {
+ out.writeFloat(f);
+ }
+
+ @Override
+ public void writeDouble(double d) throws IOException {
+ out.writeDouble(d);
+ }
+
+ @Override
+ public void writeEnum(int e) throws IOException {
+ out.writeInt(e);
+ }
+
+
+ // --------------------------------------------------------------------------------------------
+ // bytes
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void writeFixed(byte[] bytes, int start, int len) throws IOException {
+ out.write(bytes, start, len);
+ }
+
+ @Override
+ public void writeBytes(byte[] bytes, int start, int len) throws IOException {
+ out.writeInt(len);
+ if (len > 0) {
+ out.write(bytes, start, len);
+ }
+ }
+
+ @Override
+ public void writeBytes(ByteBuffer bytes) throws IOException {
+ int num = bytes.remaining();
+ out.writeInt(num);
+
+ if (num > 0) {
+ writeFixed(bytes);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // strings
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void writeString(String str) throws IOException {
+ byte[] bytes = Utf8.getBytesFor(str);
+ writeBytes(bytes, 0, bytes.length);
+ }
+
+ @Override
+ public void writeString(Utf8 utf8) throws IOException {
+ writeBytes(utf8.getBytes(), 0, utf8.getByteLength());
+
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // collection types
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void writeArrayStart() {}
+
+ @Override
+ public void setItemCount(long itemCount) throws IOException {
+ if (itemCount > 0) {
+ writeVarLongCount(out, itemCount);
+ }
+ }
+
+ @Override
+ public void startItem() {}
+
+ @Override
+ public void writeArrayEnd() throws IOException {
+ // write a single byte 0, shortcut for a var-length long of 0
+ out.write(0);
+ }
+
+ @Override
+ public void writeMapStart() {}
+
+ @Override
+ public void writeMapEnd() throws IOException {
+ // write a single byte 0, shortcut for a var-length long of 0
+ out.write(0);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // union
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void writeIndex(int unionIndex) throws IOException {
+ out.writeInt(unionIndex);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // utils
+ // --------------------------------------------------------------------------------------------
+
+
+ public static void writeVarLongCount(DataOutput out, long val) throws IOException {
+ if (val < 0) {
+ throw new IOException("Illegal count (must be non-negative): " + val);
+ }
+
+ while ((val & ~0x7FL) != 0) {
+ out.write(((int) val) | 0x80);
+ val >>>= 7;
+ }
+ out.write((int) val);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java
new file mode 100644
index 0000000..655b6ff
--- /dev/null
+++ b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.avro;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.avro.file.SeekableInput;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/**
+ * Code copy pasted from org.apache.avro.mapred.FSInput (which is Apache licensed as well)
+ *
+ * The wrapper keeps track of the position in the data stream.
+ */
+public class FSDataInputStreamWrapper implements Closeable, SeekableInput {
+ private final FSDataInputStream stream;
+ private final long len;
+ private long pos;
+
+ public FSDataInputStreamWrapper(FSDataInputStream stream, long len) {
+ this.stream = stream;
+ this.len = len;
+ this.pos = 0;
+ }
+
+ public long length() {
+ return len;
+ }
+
+ public int read(byte[] b, int off, int len) throws IOException {
+ int read;
+ read = stream.read(b, off, len);
+ pos += read;
+ return read;
+ }
+
+ public void seek(long p) throws IOException {
+ stream.seek(p);
+ pos = p;
+ }
+
+ public long tell() throws IOException {
+ return pos;
+ }
+
+ public void close() throws IOException {
+ stream.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java
new file mode 100644
index 0000000..6affeec
--- /dev/null
+++ b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.io.avro.example;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.io.GenericInputFormat;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Collector;
+
+@SuppressWarnings("serial")
+public class AvroTypeExample {
+
+
+ public static void main(String[] args) throws Exception {
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<User> users = env.createInput(new UserGeneratingInputFormat());
+
+ users
+ .map(new NumberExtractingMapper())
+ .groupBy(1)
+ .reduceGroup(new ConcatenatingReducer())
+ .print();
+
+ env.execute();
+ }
+
+
+
+ public static final class NumberExtractingMapper implements MapFunction<User, Tuple2<User, Integer>> {
+
+ @Override
+ public Tuple2<User, Integer> map(User user) {
+ return new Tuple2<User, Integer>(user, user.getFavoriteNumber());
+ }
+ }
+
+
+ public static final class ConcatenatingReducer implements GroupReduceFunction<Tuple2<User, Integer>, Tuple2<Integer, String>> {
+
+ @Override
+ public void reduce(Iterable<Tuple2<User, Integer>> values, Collector<Tuple2<Integer, String>> out) throws Exception {
+ int number = 0;
+ StringBuilder colors = new StringBuilder();
+
+ for (Tuple2<User, Integer> u : values) {
+ number = u.f1;
+ colors.append(u.f0.getFavoriteColor()).append(" - ");
+ }
+
+ colors.setLength(colors.length() - 3);
+ out.collect(new Tuple2<Integer, String>(number, colors.toString()));
+ }
+ }
+
+
+ public static final class UserGeneratingInputFormat extends GenericInputFormat<User> {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final int NUM = 100;
+
+ private final Random rnd = new Random(32498562304986L);
+
+ private static final String[] NAMES = { "Peter", "Bob", "Liddy", "Alexander", "Stan" };
+
+ private static final String[] COLORS = { "mauve", "crimson", "copper", "sky", "grass" };
+
+ private int count;
+
+
+ @Override
+ public boolean reachedEnd() throws IOException {
+ return count >= NUM;
+ }
+
+ @Override
+ public User nextRecord(User reuse) throws IOException {
+ count++;
+
+ User u = new User();
+ u.setName(NAMES[rnd.nextInt(NAMES.length)]);
+ u.setFavoriteColor(COLORS[rnd.nextInt(COLORS.length)]);
+ u.setFavoriteNumber(rnd.nextInt(87));
+ return u;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java
new file mode 100644
index 0000000..4608f96
--- /dev/null
+++ b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * Autogenerated by Avro
+ *
+ * DO NOT EDIT DIRECTLY
+ */
+package org.apache.flink.api.io.avro.example;
+@SuppressWarnings("all")
+@org.apache.avro.specific.AvroGenerated
+public class User extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"org.apache.flink.api.avro.example\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}");
+ public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
+ @Deprecated public java.lang.CharSequence name;
+ @Deprecated public java.lang.Integer favorite_number;
+ @Deprecated public java.lang.CharSequence favorite_color;
+
+ /**
+ * Default constructor. Note that this does not initialize fields
+ * to their default values from the schema. If that is desired then
+ * one should use {@link #newBuilder()}.
+ */
+ public User() {}
+
+ /**
+ * All-args constructor.
+ */
+ public User(java.lang.CharSequence name, java.lang.Integer favorite_number, java.lang.CharSequence favorite_color) {
+ this.name = name;
+ this.favorite_number = favorite_number;
+ this.favorite_color = favorite_color;
+ }
+
+ public org.apache.avro.Schema getSchema() { return SCHEMA$; }
+ // Used by DatumWriter. Applications should not call.
+ public java.lang.Object get(int field$) {
+ switch (field$) {
+ case 0: return name;
+ case 1: return favorite_number;
+ case 2: return favorite_color;
+ default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+ }
+ }
+ // Used by DatumReader. Applications should not call.
+ @SuppressWarnings(value="unchecked")
+ public void put(int field$, java.lang.Object value$) {
+ switch (field$) {
+ case 0: name = (java.lang.CharSequence)value$; break;
+ case 1: favorite_number = (java.lang.Integer)value$; break;
+ case 2: favorite_color = (java.lang.CharSequence)value$; break;
+ default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+ }
+ }
+
+ /**
+ * Gets the value of the 'name' field.
+ */
+ public java.lang.CharSequence getName() {
+ return name;
+ }
+
+ /**
+ * Sets the value of the 'name' field.
+ * @param value the value to set.
+ */
+ public void setName(java.lang.CharSequence value) {
+ this.name = value;
+ }
+
+ /**
+ * Gets the value of the 'favorite_number' field.
+ */
+ public java.lang.Integer getFavoriteNumber() {
+ return favorite_number;
+ }
+
+ /**
+ * Sets the value of the 'favorite_number' field.
+ * @param value the value to set.
+ */
+ public void setFavoriteNumber(java.lang.Integer value) {
+ this.favorite_number = value;
+ }
+
+ /**
+ * Gets the value of the 'favorite_color' field.
+ */
+ public java.lang.CharSequence getFavoriteColor() {
+ return favorite_color;
+ }
+
+ /**
+ * Sets the value of the 'favorite_color' field.
+ * @param value the value to set.
+ */
+ public void setFavoriteColor(java.lang.CharSequence value) {
+ this.favorite_color = value;
+ }
+
+ /** Creates a new User RecordBuilder */
+ public static org.apache.flink.api.io.avro.example.User.Builder newBuilder() {
+ return new org.apache.flink.api.io.avro.example.User.Builder();
+ }
+
+ /** Creates a new User RecordBuilder by copying an existing Builder */
+ public static org.apache.flink.api.io.avro.example.User.Builder newBuilder(org.apache.flink.api.io.avro.example.User.Builder other) {
+ return new org.apache.flink.api.io.avro.example.User.Builder(other);
+ }
+
+ /** Creates a new User RecordBuilder by copying an existing User instance */
+ public static org.apache.flink.api.io.avro.example.User.Builder newBuilder(org.apache.flink.api.io.avro.example.User other) {
+ return new org.apache.flink.api.io.avro.example.User.Builder(other);
+ }
+
+ /**
+ * RecordBuilder for User instances.
+ */
+ public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<User>
+ implements org.apache.avro.data.RecordBuilder<User> {
+
+ private java.lang.CharSequence name;
+ private java.lang.Integer favorite_number;
+ private java.lang.CharSequence favorite_color;
+
+ /** Creates a new Builder */
+ private Builder() {
+ super(org.apache.flink.api.io.avro.example.User.SCHEMA$);
+ }
+
+ /** Creates a Builder by copying an existing Builder */
+ private Builder(org.apache.flink.api.io.avro.example.User.Builder other) {
+ super(other);
+ if (isValidValue(fields()[0], other.name)) {
+ this.name = data().deepCopy(fields()[0].schema(), other.name);
+ fieldSetFlags()[0] = true;
+ }
+ if (isValidValue(fields()[1], other.favorite_number)) {
+ this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);
+ fieldSetFlags()[1] = true;
+ }
+ if (isValidValue(fields()[2], other.favorite_color)) {
+ this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);
+ fieldSetFlags()[2] = true;
+ }
+ }
+
+ /** Creates a Builder by copying an existing User instance */
+ private Builder(org.apache.flink.api.io.avro.example.User other) {
+ super(org.apache.flink.api.io.avro.example.User.SCHEMA$);
+ if (isValidValue(fields()[0], other.name)) {
+ this.name = data().deepCopy(fields()[0].schema(), other.name);
+ fieldSetFlags()[0] = true;
+ }
+ if (isValidValue(fields()[1], other.favorite_number)) {
+ this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);
+ fieldSetFlags()[1] = true;
+ }
+ if (isValidValue(fields()[2], other.favorite_color)) {
+ this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);
+ fieldSetFlags()[2] = true;
+ }
+ }
+
+ /** Gets the value of the 'name' field */
+ public java.lang.CharSequence getName() {
+ return name;
+ }
+
+ /** Sets the value of the 'name' field */
+ public org.apache.flink.api.io.avro.example.User.Builder setName(java.lang.CharSequence value) {
+ validate(fields()[0], value);
+ this.name = value;
+ fieldSetFlags()[0] = true;
+ return this;
+ }
+
+ /** Checks whether the 'name' field has been set */
+ public boolean hasName() {
+ return fieldSetFlags()[0];
+ }
+
+ /** Clears the value of the 'name' field */
+ public org.apache.flink.api.io.avro.example.User.Builder clearName() {
+ name = null;
+ fieldSetFlags()[0] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'favorite_number' field */
+ public java.lang.Integer getFavoriteNumber() {
+ return favorite_number;
+ }
+
+ /** Sets the value of the 'favorite_number' field */
+ public org.apache.flink.api.io.avro.example.User.Builder setFavoriteNumber(java.lang.Integer value) {
+ validate(fields()[1], value);
+ this.favorite_number = value;
+ fieldSetFlags()[1] = true;
+ return this;
+ }
+
+ /** Checks whether the 'favorite_number' field has been set */
+ public boolean hasFavoriteNumber() {
+ return fieldSetFlags()[1];
+ }
+
+ /** Clears the value of the 'favorite_number' field */
+ public org.apache.flink.api.io.avro.example.User.Builder clearFavoriteNumber() {
+ favorite_number = null;
+ fieldSetFlags()[1] = false;
+ return this;
+ }
+
+ /** Gets the value of the 'favorite_color' field */
+ public java.lang.CharSequence getFavoriteColor() {
+ return favorite_color;
+ }
+
+ /** Sets the value of the 'favorite_color' field */
+ public org.apache.flink.api.io.avro.example.User.Builder setFavoriteColor(java.lang.CharSequence value) {
+ validate(fields()[2], value);
+ this.favorite_color = value;
+ fieldSetFlags()[2] = true;
+ return this;
+ }
+
+ /** Checks whether the 'favorite_color' field has been set */
+ public boolean hasFavoriteColor() {
+ return fieldSetFlags()[2];
+ }
+
+ /** Clears the value of the 'favorite_color' field */
+ public org.apache.flink.api.io.avro.example.User.Builder clearFavoriteColor() {
+ favorite_color = null;
+ fieldSetFlags()[2] = false;
+ return this;
+ }
+
+ @Override
+ public User build() {
+ try {
+ User record = new User();
+ record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]);
+ record.favorite_number = fieldSetFlags()[1] ? this.favorite_number : (java.lang.Integer) defaultValue(fields()[1]);
+ record.favorite_color = fieldSetFlags()[2] ? this.favorite_color : (java.lang.CharSequence) defaultValue(fields()[2]);
+ return record;
+ } catch (Exception e) {
+ throw new org.apache.avro.AvroRuntimeException(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java
new file mode 100644
index 0000000..fbb0f4c
--- /dev/null
+++ b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.java.io;
+
+import java.io.IOException;
+
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.file.FileReader;
+import org.apache.avro.file.SeekableInput;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.reflect.ReflectDatumReader;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.flink.api.avro.FSDataInputStreamWrapper;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.core.fs.FileInputSplit;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.util.InstantiationUtil;
+
+
+public class AvroInputFormat<E> extends FileInputFormat<E> implements ResultTypeQueryable<E> {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(AvroInputFormat.class);
+
+
+ private final Class<E> avroValueType;
+
+ private boolean reuseAvroValue = true;
+
+
+ private transient FileReader<E> dataFileReader;
+
+
+ public AvroInputFormat(Path filePath, Class<E> type) {
+ super(filePath);
+ this.avroValueType = type;
+ this.unsplittable = true;
+ }
+
+
+ /**
+ * Sets the flag whether to reuse the Avro value instance for all records.
+ * By default, the input format reuses the Avro value.
+ *
+ * @param reuseAvroValue True, if the input format should reuse the Avro value instance, false otherwise.
+ */
+ public void setReuseAvroValue(boolean reuseAvroValue) {
+ this.reuseAvroValue = reuseAvroValue;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Typing
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public TypeInformation<E> getProducedType() {
+ return TypeExtractor.getForClass(this.avroValueType);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Input Format Methods
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void open(FileInputSplit split) throws IOException {
+ super.open(split);
+
+ DatumReader<E> datumReader;
+ if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) {
+ datumReader = new SpecificDatumReader<E>(avroValueType);
+ } else {
+ datumReader = new ReflectDatumReader<E>(avroValueType);
+ }
+
+ LOG.info("Opening split " + split);
+
+ SeekableInput in = new FSDataInputStreamWrapper(stream, (int) split.getLength());
+
+ dataFileReader = DataFileReader.openReader(in, datumReader);
+ dataFileReader.sync(split.getStart());
+ }
+
+ @Override
+ public boolean reachedEnd() throws IOException {
+ return !dataFileReader.hasNext();
+ }
+
+ @Override
+ public E nextRecord(E reuseValue) throws IOException {
+ if (!dataFileReader.hasNext()) {
+ return null;
+ }
+
+ if (!reuseAvroValue) {
+ reuseValue = InstantiationUtil.instantiate(avroValueType, Object.class);
+ }
+
+ reuseValue = dataFileReader.next(reuseValue);
+ return reuseValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java
new file mode 100644
index 0000000..d00dbf7
--- /dev/null
+++ b/flink-staging/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroOutputFormat.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.java.io;
+
+
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.reflect.ReflectDatumWriter;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.flink.api.common.io.FileOutputFormat;
+import org.apache.flink.core.fs.Path;
+
+import java.io.IOException;
+
+public class AvroOutputFormat<E> extends FileOutputFormat<E> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Class<E> avroValueType;
+
+ private Schema userDefinedSchema = null;
+
+ private transient DataFileWriter<E> dataFileWriter;
+
+ public AvroOutputFormat(Path filePath, Class<E> type) {
+ super(filePath);
+ this.avroValueType = type;
+ }
+
+ public AvroOutputFormat(Class<E> type) {
+ this.avroValueType = type;
+ }
+
+ @Override
+ protected String getDirectoryFileName(int taskNumber) {
+ return super.getDirectoryFileName(taskNumber) + ".avro";
+ }
+
+ public void setSchema(Schema schema) {
+ this.userDefinedSchema = schema;
+ }
+
+ @Override
+ public void writeRecord(E record) throws IOException {
+ dataFileWriter.append(record);
+ }
+
+ @Override
+ public void open(int taskNumber, int numTasks) throws IOException {
+ super.open(taskNumber, numTasks);
+
+ DatumWriter<E> datumWriter;
+ Schema schema = null;
+ if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) {
+ datumWriter = new SpecificDatumWriter<E>(avroValueType);
+ try {
+ schema = ((org.apache.avro.specific.SpecificRecordBase)avroValueType.newInstance()).getSchema();
+ } catch (InstantiationException e) {
+ throw new RuntimeException(e.getMessage());
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+ } else {
+ datumWriter = new ReflectDatumWriter<E>(avroValueType);
+ schema = ReflectData.get().getSchema(avroValueType);
+ }
+ dataFileWriter = new DataFileWriter<E>(datumWriter);
+ if (userDefinedSchema == null) {
+ dataFileWriter.create(schema, stream);
+ } else {
+ dataFileWriter.create(userDefinedSchema, stream);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ dataFileWriter.flush();
+ dataFileWriter.close();
+ super.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/assembly/test-assembly.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/assembly/test-assembly.xml b/flink-staging/flink-avro/src/test/assembly/test-assembly.xml
new file mode 100644
index 0000000..0f4561a
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/assembly/test-assembly.xml
@@ -0,0 +1,36 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<assembly>
+ <id>test-jar</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.testOutputDirectory}</directory>
+ <outputDirectory>/</outputDirectory>
+ <!--modify/add include to match your package(s) -->
+ <includes>
+ <include>org/apache/flink/api/avro/testjar/**</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java
new file mode 100644
index 0000000..0b394e3
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/AvroExternalJarProgramITCase.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.avro;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+
+import org.apache.flink.client.program.Client;
+import org.apache.flink.client.program.PackagedProgram;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.test.util.ForkableFlinkMiniCluster;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class AvroExternalJarProgramITCase {
+
+ private static final String JAR_FILE = "target/maven-test-jar.jar";
+
+ private static final String TEST_DATA_FILE = "/testdata.avro";
+
+ @Test
+ public void testExternalProgram() {
+
+ ForkableFlinkMiniCluster testMiniCluster = null;
+
+ try {
+ Configuration config = new Configuration();
+ config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 4);
+ testMiniCluster = new ForkableFlinkMiniCluster(config, false);
+
+ String jarFile = JAR_FILE;
+ String testData = getClass().getResource(TEST_DATA_FILE).toString();
+
+ PackagedProgram program = new PackagedProgram(new File(jarFile), new String[] { testData });
+
+ Client c = new Client(new InetSocketAddress("localhost", testMiniCluster.getJobManagerRPCPort()),
+ new Configuration(), program.getUserCodeClassLoader());
+ c.run(program, 4, true);
+ }
+ catch (Throwable t) {
+ System.err.println(t.getMessage());
+ t.printStackTrace();
+ Assert.fail("Error during the packaged program execution: " + t.getMessage());
+ }
+ finally {
+ if (testMiniCluster != null) {
+ try {
+ testMiniCluster.stop();
+ } catch (Throwable t) {}
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java
new file mode 100644
index 0000000..a8bace3
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.avro;
+
+import org.junit.Assert;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.reflect.ReflectDatumReader;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.flink.api.io.avro.example.User;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.java.io.AvroOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.test.util.JavaProgramTestBase;
+
+@SuppressWarnings("serial")
+public class AvroOutputFormatTest extends JavaProgramTestBase {
+
+ public static String outputPath1;
+
+ public static String outputPath2;
+
+ public static String inputPath;
+
+ public static String userData = "alice|1|blue\n" +
+ "bob|2|red\n" +
+ "john|3|yellow\n" +
+ "walt|4|black\n";
+
+ @Override
+ protected void preSubmit() throws Exception {
+ inputPath = createTempFile("user", userData);
+ outputPath1 = getTempDirPath("avro_output1");
+ outputPath2 = getTempDirPath("avro_output2");
+ }
+
+
+ @Override
+ protected void testProgram() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<String, Integer, String>> input = env.readCsvFile(inputPath)
+ .fieldDelimiter("|")
+ .types(String.class, Integer.class, String.class);
+
+ //output the data with AvroOutputFormat for specific user type
+ DataSet<User> specificUser = input.map(new ConvertToUser());
+ specificUser.write(new AvroOutputFormat<User>(User.class), outputPath1);
+
+ //output the data with AvroOutputFormat for reflect user type
+ DataSet<ReflectiveUser> reflectiveUser = specificUser.map(new ConvertToReflective());
+ reflectiveUser.write(new AvroOutputFormat<ReflectiveUser>(ReflectiveUser.class), outputPath2);
+
+ env.execute();
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ //compare result for specific user type
+ File [] output1;
+ File file1 = asFile(outputPath1);
+ if (file1.isDirectory()) {
+ output1 = file1.listFiles();
+ // check for avro ext in dir.
+ for (File avroOutput : output1) {
+ Assert.assertTrue("Expect extension '.avro'", avroOutput.toString().endsWith(".avro"));
+ }
+ } else {
+ output1 = new File[] {file1};
+ }
+ List<String> result1 = new ArrayList<String>();
+ DatumReader<User> userDatumReader1 = new SpecificDatumReader<User>(User.class);
+ for (File avroOutput : output1) {
+
+ DataFileReader<User> dataFileReader1 = new DataFileReader<User>(avroOutput, userDatumReader1);
+ while (dataFileReader1.hasNext()) {
+ User user = dataFileReader1.next();
+ result1.add(user.getName() + "|" + user.getFavoriteNumber() + "|" + user.getFavoriteColor());
+ }
+ }
+ for (String expectedResult : userData.split("\n")) {
+ Assert.assertTrue("expected user " + expectedResult + " not found.", result1.contains(expectedResult));
+ }
+
+ //compare result for reflect user type
+ File [] output2;
+ File file2 = asFile(outputPath2);
+ if (file2.isDirectory()) {
+ output2 = file2.listFiles();
+ } else {
+ output2 = new File[] {file2};
+ }
+ List<String> result2 = new ArrayList<String>();
+ DatumReader<ReflectiveUser> userDatumReader2 = new ReflectDatumReader<ReflectiveUser>(ReflectiveUser.class);
+ for (File avroOutput : output2) {
+ DataFileReader<ReflectiveUser> dataFileReader2 = new DataFileReader<ReflectiveUser>(avroOutput, userDatumReader2);
+ while (dataFileReader2.hasNext()) {
+ ReflectiveUser user = dataFileReader2.next();
+ result2.add(user.getName() + "|" + user.getFavoriteNumber() + "|" + user.getFavoriteColor());
+ }
+ }
+ for (String expectedResult : userData.split("\n")) {
+ Assert.assertTrue("expected user " + expectedResult + " not found.", result2.contains(expectedResult));
+ }
+
+
+ }
+
+
+ public final static class ConvertToUser extends RichMapFunction<Tuple3<String, Integer, String>, User> {
+
+ @Override
+ public User map(Tuple3<String, Integer, String> value) throws Exception {
+ return new User(value.f0, value.f1, value.f2);
+ }
+ }
+
+ public final static class ConvertToReflective extends RichMapFunction<User, ReflectiveUser> {
+
+ @Override
+ public ReflectiveUser map(User value) throws Exception {
+ return new ReflectiveUser(value.getName().toString(), value.getFavoriteNumber(), value.getFavoriteColor().toString());
+ }
+ }
+
+
+ public static class ReflectiveUser {
+ private String name;
+ private int favoriteNumber;
+ private String favoriteColor;
+
+ public ReflectiveUser() {}
+
+ public ReflectiveUser(String name, int favoriteNumber, String favoriteColor) {
+ this.name = name;
+ this.favoriteNumber = favoriteNumber;
+ this.favoriteColor = favoriteColor;
+ }
+
+ public String getName() {
+ return this.name;
+ }
+ public String getFavoriteColor() {
+ return this.favoriteColor;
+ }
+ public int getFavoriteNumber() {
+ return this.favoriteNumber;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/EncoderDecoderTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/EncoderDecoderTest.java b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/EncoderDecoderTest.java
new file mode 100644
index 0000000..0724457
--- /dev/null
+++ b/flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/EncoderDecoderTest.java
@@ -0,0 +1,519 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.avro;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.avro.reflect.ReflectDatumReader;
+import org.apache.avro.reflect.ReflectDatumWriter;
+import org.apache.flink.api.io.avro.generated.Colors;
+import org.apache.flink.api.io.avro.generated.User;
+import org.apache.flink.util.StringUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * Tests the {@link DataOutputEncoder} and {@link DataInputDecoder} classes for Avro serialization.
+ */
+public class EncoderDecoderTest {
+ @Test
+ public void testComplexStringsDirecty() {
+ try {
+ Random rnd = new Random(349712539451944123L);
+
+ for (int i = 0; i < 10; i++) {
+ String testString = StringUtils.getRandomString(rnd, 10, 100);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
+ {
+ DataOutputStream dataOut = new DataOutputStream(baos);
+ DataOutputEncoder encoder = new DataOutputEncoder();
+ encoder.setOut(dataOut);
+
+ encoder.writeString(testString);
+ dataOut.flush();
+ dataOut.close();
+ }
+
+ byte[] data = baos.toByteArray();
+
+ // deserialize
+ {
+ ByteArrayInputStream bais = new ByteArrayInputStream(data);
+ DataInputStream dataIn = new DataInputStream(bais);
+ DataInputDecoder decoder = new DataInputDecoder();
+ decoder.setIn(dataIn);
+
+ String deserialized = decoder.readString();
+
+ assertEquals(testString, deserialized);
+ }
+ }
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test failed due to an exception: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPrimitiveTypes() {
+
+ testObjectSerialization(new Boolean(true));
+ testObjectSerialization(new Boolean(false));
+
+ testObjectSerialization(Byte.valueOf((byte) 0));
+ testObjectSerialization(Byte.valueOf((byte) 1));
+ testObjectSerialization(Byte.valueOf((byte) -1));
+ testObjectSerialization(Byte.valueOf(Byte.MIN_VALUE));
+ testObjectSerialization(Byte.valueOf(Byte.MAX_VALUE));
+
+ testObjectSerialization(Short.valueOf((short) 0));
+ testObjectSerialization(Short.valueOf((short) 1));
+ testObjectSerialization(Short.valueOf((short) -1));
+ testObjectSerialization(Short.valueOf(Short.MIN_VALUE));
+ testObjectSerialization(Short.valueOf(Short.MAX_VALUE));
+
+ testObjectSerialization(Integer.valueOf(0));
+ testObjectSerialization(Integer.valueOf(1));
+ testObjectSerialization(Integer.valueOf(-1));
+ testObjectSerialization(Integer.valueOf(Integer.MIN_VALUE));
+ testObjectSerialization(Integer.valueOf(Integer.MAX_VALUE));
+
+ testObjectSerialization(Long.valueOf(0));
+ testObjectSerialization(Long.valueOf(1));
+ testObjectSerialization(Long.valueOf(-1));
+ testObjectSerialization(Long.valueOf(Long.MIN_VALUE));
+ testObjectSerialization(Long.valueOf(Long.MAX_VALUE));
+
+ testObjectSerialization(Float.valueOf(0));
+ testObjectSerialization(Float.valueOf(1));
+ testObjectSerialization(Float.valueOf(-1));
+ testObjectSerialization(Float.valueOf((float)Math.E));
+ testObjectSerialization(Float.valueOf((float)Math.PI));
+ testObjectSerialization(Float.valueOf(Float.MIN_VALUE));
+ testObjectSerialization(Float.valueOf(Float.MAX_VALUE));
+ testObjectSerialization(Float.valueOf(Float.MIN_NORMAL));
+ testObjectSerialization(Float.valueOf(Float.NaN));
+ testObjectSerialization(Float.valueOf(Float.NEGATIVE_INFINITY));
+ testObjectSerialization(Float.valueOf(Float.POSITIVE_INFINITY));
+
+ testObjectSerialization(Double.valueOf(0));
+ testObjectSerialization(Double.valueOf(1));
+ testObjectSerialization(Double.valueOf(-1));
+ testObjectSerialization(Double.valueOf(Math.E));
+ testObjectSerialization(Double.valueOf(Math.PI));
+ testObjectSerialization(Double.valueOf(Double.MIN_VALUE));
+ testObjectSerialization(Double.valueOf(Double.MAX_VALUE));
+ testObjectSerialization(Double.valueOf(Double.MIN_NORMAL));
+ testObjectSerialization(Double.valueOf(Double.NaN));
+ testObjectSerialization(Double.valueOf(Double.NEGATIVE_INFINITY));
+ testObjectSerialization(Double.valueOf(Double.POSITIVE_INFINITY));
+
+ testObjectSerialization("");
+ testObjectSerialization("abcdefg");
+ testObjectSerialization("ab\u1535\u0155xyz\u706F");
+
+ testObjectSerialization(new SimpleTypes(3637, 54876486548L, (byte) 65, "We're out looking for astronauts", (short) 0x2387, 2.65767523));
+ testObjectSerialization(new SimpleTypes(705608724, -1L, (byte) -65, "Serve me the sky with a big slice of lemon", (short) Byte.MIN_VALUE, 0.0000001));
+ }
+
+ @Test
+ public void testArrayTypes() {
+ {
+ int[] array = new int[] {1, 2, 3, 4, 5};
+ testObjectSerialization(array);
+ }
+ {
+ long[] array = new long[] {1, 2, 3, 4, 5};
+ testObjectSerialization(array);
+ }
+ {
+ float[] array = new float[] {1, 2, 3, 4, 5};
+ testObjectSerialization(array);
+ }
+ {
+ double[] array = new double[] {1, 2, 3, 4, 5};
+ testObjectSerialization(array);
+ }
+ {
+ String[] array = new String[] {"Oh", "my", "what", "do", "we", "have", "here", "?"};
+ testObjectSerialization(array);
+ }
+ }
+
+ @Test
+ public void testEmptyArray() {
+ {
+ int[] array = new int[0];
+ testObjectSerialization(array);
+ }
+ {
+ long[] array = new long[0];
+ testObjectSerialization(array);
+ }
+ {
+ float[] array = new float[0];
+ testObjectSerialization(array);
+ }
+ {
+ double[] array = new double[0];
+ testObjectSerialization(array);
+ }
+ {
+ String[] array = new String[0];
+ testObjectSerialization(array);
+ }
+ }
+
+ @Test
+ public void testObjects() {
+ // simple object containing only primitives
+ {
+ testObjectSerialization(new Book(976243875L, "The Serialization Odysse", 42));
+ }
+
+ // object with collection
+ {
+ ArrayList<String> list = new ArrayList<String>();
+ list.add("A");
+ list.add("B");
+ list.add("C");
+ list.add("D");
+ list.add("E");
+
+ testObjectSerialization(new BookAuthor(976243875L, list, "Arno Nym"));
+ }
+
+ // object with empty collection
+ {
+ ArrayList<String> list = new ArrayList<String>();
+ testObjectSerialization(new BookAuthor(987654321L, list, "The Saurus"));
+ }
+ }
+
+ @Test
+ public void testNestedObjectsWithCollections() {
+ testObjectSerialization(new ComplexNestedObject2(true));
+ }
+
+ @Test
+ public void testGeneratedObjectWithNullableFields() {
+ List<CharSequence> strings = Arrays.asList(new CharSequence[] { "These", "strings", "should", "be", "recognizable", "as", "a", "meaningful", "sequence" });
+ List<Boolean> bools = Arrays.asList(true, true, false, false, true, false, true, true);
+ Map<CharSequence, Long> map = new HashMap<CharSequence, Long>();
+ map.put("1", 1L);
+ map.put("2", 2L);
+ map.put("3", 3L);
+
+ User user = new User("Freudenreich", 1337, "macintosh gray", 1234567890L, 3.1415926, null, true, strings, bools, null, Colors.GREEN, map);
+
+ testObjectSerialization(user);
+ }
+
+ @Test
+ public void testVarLenCountEncoding() {
+ try {
+ long[] values = new long[] { 0, 1, 2, 3, 4, 0, 574, 45236, 0, 234623462, 23462462346L, 0, 9734028767869761L, 0x7fffffffffffffffL};
+
+ // write
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
+ {
+ DataOutputStream dataOut = new DataOutputStream(baos);
+
+ for (long val : values) {
+ DataOutputEncoder.writeVarLongCount(dataOut, val);
+ }
+
+ dataOut.flush();
+ dataOut.close();
+ }
+
+ // read
+ {
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dataIn = new DataInputStream(bais);
+
+ for (long val : values) {
+ long read = DataInputDecoder.readVarLongCount(dataIn);
+ assertEquals("Wrong var-len encoded value read.", val, read);
+ }
+ }
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test failed due to an exception: " + e.getMessage());
+ }
+ }
+
+ private static <X> void testObjectSerialization(X obj) {
+
+ try {
+
+ // serialize
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
+ {
+ DataOutputStream dataOut = new DataOutputStream(baos);
+ DataOutputEncoder encoder = new DataOutputEncoder();
+ encoder.setOut(dataOut);
+
+ @SuppressWarnings("unchecked")
+ Class<X> clazz = (Class<X>) obj.getClass();
+ ReflectDatumWriter<X> writer = new ReflectDatumWriter<X>(clazz);
+
+ writer.write(obj, encoder);
+ dataOut.flush();
+ dataOut.close();
+ }
+
+ byte[] data = baos.toByteArray();
+ X result = null;
+
+ // deserialize
+ {
+ ByteArrayInputStream bais = new ByteArrayInputStream(data);
+ DataInputStream dataIn = new DataInputStream(bais);
+ DataInputDecoder decoder = new DataInputDecoder();
+ decoder.setIn(dataIn);
+
+ @SuppressWarnings("unchecked")
+ Class<X> clazz = (Class<X>) obj.getClass();
+ ReflectDatumReader<X> reader = new ReflectDatumReader<X>(clazz);
+
+ // create a reuse object if possible, otherwise we have no reuse object
+ X reuse = null;
+ try {
+ @SuppressWarnings("unchecked")
+ X test = (X) obj.getClass().newInstance();
+ reuse = test;
+ } catch (Throwable t) {}
+
+ result = reader.read(reuse, decoder);
+ }
+
+ // check
+ final String message = "Deserialized object is not the same as the original";
+
+ if (obj.getClass().isArray()) {
+ Class<?> clazz = obj.getClass();
+ if (clazz == byte[].class) {
+ assertArrayEquals(message, (byte[]) obj, (byte[]) result);
+ }
+ else if (clazz == short[].class) {
+ assertArrayEquals(message, (short[]) obj, (short[]) result);
+ }
+ else if (clazz == int[].class) {
+ assertArrayEquals(message, (int[]) obj, (int[]) result);
+ }
+ else if (clazz == long[].class) {
+ assertArrayEquals(message, (long[]) obj, (long[]) result);
+ }
+ else if (clazz == char[].class) {
+ assertArrayEquals(message, (char[]) obj, (char[]) result);
+ }
+ else if (clazz == float[].class) {
+ assertArrayEquals(message, (float[]) obj, (float[]) result, 0.0f);
+ }
+ else if (clazz == double[].class) {
+ assertArrayEquals(message, (double[]) obj, (double[]) result, 0.0);
+ } else {
+ assertArrayEquals(message, (Object[]) obj, (Object[]) result);
+ }
+ } else {
+ assertEquals(message, obj, result);
+ }
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test failed due to an exception: " + e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Test Objects
+ // --------------------------------------------------------------------------------------------
+
+
+ public static final class SimpleTypes {
+
+ private final int iVal;
+ private final long lVal;
+ private final byte bVal;
+ private final String sVal;
+ private final short rVal;
+ private final double dVal;
+
+
+ public SimpleTypes() {
+ this(0, 0, (byte) 0, "", (short) 0, 0);
+ }
+
+ public SimpleTypes(int iVal, long lVal, byte bVal, String sVal, short rVal, double dVal) {
+ this.iVal = iVal;
+ this.lVal = lVal;
+ this.bVal = bVal;
+ this.sVal = sVal;
+ this.rVal = rVal;
+ this.dVal = dVal;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() == SimpleTypes.class) {
+ SimpleTypes other = (SimpleTypes) obj;
+
+ return other.iVal == this.iVal &&
+ other.lVal == this.lVal &&
+ other.bVal == this.bVal &&
+ other.sVal.equals(this.sVal) &&
+ other.rVal == this.rVal &&
+ other.dVal == this.dVal;
+
+ } else {
+ return false;
+ }
+ }
+ }
+
+ public static class ComplexNestedObject1 {
+
+ private double doubleValue;
+
+ private List<String> stringList;
+
+ public ComplexNestedObject1() {}
+
+ public ComplexNestedObject1(int offInit) {
+ this.doubleValue = 6293485.6723 + offInit;
+
+ this.stringList = new ArrayList<String>();
+ this.stringList.add("A" + offInit);
+ this.stringList.add("somewhat" + offInit);
+ this.stringList.add("random" + offInit);
+ this.stringList.add("collection" + offInit);
+ this.stringList.add("of" + offInit);
+ this.stringList.add("strings" + offInit);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() == ComplexNestedObject1.class) {
+ ComplexNestedObject1 other = (ComplexNestedObject1) obj;
+ return other.doubleValue == this.doubleValue && this.stringList.equals(other.stringList);
+ } else {
+ return false;
+ }
+ }
+ }
+
+ public static class ComplexNestedObject2 {
+
+ private long longValue;
+
+ private Map<String, ComplexNestedObject1> theMap;
+
+ public ComplexNestedObject2() {}
+
+ public ComplexNestedObject2(boolean init) {
+ this.longValue = 46547;
+
+ this.theMap = new HashMap<String, ComplexNestedObject1>();
+ this.theMap.put("36354L", new ComplexNestedObject1(43546543));
+ this.theMap.put("785611L", new ComplexNestedObject1(45784568));
+ this.theMap.put("43L", new ComplexNestedObject1(9876543));
+ this.theMap.put("-45687L", new ComplexNestedObject1(7897615));
+ this.theMap.put("1919876876896L", new ComplexNestedObject1(27154));
+ this.theMap.put("-868468468L", new ComplexNestedObject1(546435));
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() == ComplexNestedObject2.class) {
+ ComplexNestedObject2 other = (ComplexNestedObject2) obj;
+ return other.longValue == this.longValue && this.theMap.equals(other.theMap);
+ } else {
+ return false;
+ }
+ }
+ }
+
+ public static class Book {
+
+ private long bookId;
+ private String title;
+ private long authorId;
+
+ public Book() {}
+
+ public Book(long bookId, String title, long authorId) {
+ this.bookId = bookId;
+ this.title = title;
+ this.authorId = authorId;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() == Book.class) {
+ Book other = (Book) obj;
+ return other.bookId == this.bookId && other.authorId == this.authorId && this.title.equals(other.title);
+ } else {
+ return false;
+ }
+ }
+ }
+
+ public static class BookAuthor {
+
+ private long authorId;
+ private List<String> bookTitles;
+ private String authorName;
+
+ public BookAuthor() {}
+
+ public BookAuthor(long authorId, List<String> bookTitles, String authorName) {
+ this.authorId = authorId;
+ this.bookTitles = bookTitles;
+ this.authorName = authorName;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() == BookAuthor.class) {
+ BookAuthor other = (BookAuthor) obj;
+ return other.authorName.equals(this.authorName) && other.authorId == this.authorId &&
+ other.bookTitles.equals(this.bookTitles);
+ } else {
+ return false;
+ }
+ }
+ }
+}
[47/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopInputFormatTest.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopInputFormatTest.java
deleted file mode 100644
index 00fd1f9..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopInputFormatTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.test.hadoopcompatibility.mapred;
-
-
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.hadoopcompatibility.mapred.HadoopInputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.junit.Assert.fail;
-
-
-public class HadoopInputFormatTest {
-
-
- public class DummyVoidKeyInputFormat<T> extends FileInputFormat<Void, T> {
-
- public DummyVoidKeyInputFormat() {
- }
-
- @Override
- public org.apache.hadoop.mapred.RecordReader<Void, T> getRecordReader(org.apache.hadoop.mapred.InputSplit inputSplit, JobConf jobConf, Reporter reporter) throws IOException {
- return null;
- }
- }
-
-
- @Test
- public void checkTypeInformation() {
- try {
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- // Set up the Hadoop Input Format
- Job job = Job.getInstance();
- HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<Void, Long>( new DummyVoidKeyInputFormat(), Void.class, Long.class, new JobConf());
-
- TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
- TypeInformation<Tuple2<Void,Long>> testTupleType = new TupleTypeInfo<Tuple2<Void,Long>>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
-
- if(tupleType.isTupleType()) {
- if(!((TupleTypeInfo)tupleType).equals(testTupleType)) {
- fail("Tuple type information was not set correctly!");
- }
- } else {
- fail("Type information was not set to tuple type information!");
- }
-
- }
- catch (Exception ex) {
- fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapFunctionITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapFunctionITCase.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapFunctionITCase.java
deleted file mode 100644
index 850e799..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapFunctionITCase.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.hadoopcompatibility.mapred;
-
-import java.io.IOException;
-
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.hadoopcompatibility.mapred.HadoopMapFunction;
-import org.apache.flink.test.util.MultipleProgramsTestBase;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class HadoopMapFunctionITCase extends MultipleProgramsTestBase {
-
- public HadoopMapFunctionITCase(ExecutionMode mode){
- super(mode);
- }
-
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
-
- @Test
- public void testNonPassingMapper() throws Exception{
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
- DataSet<Tuple2<IntWritable, Text>> nonPassingFlatMapDs = ds.
- flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new NonPassingMapper()));
-
- String resultPath = tempFolder.newFile().toURI().toString();
-
- nonPassingFlatMapDs.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
- env.execute();
-
- compareResultsByLinesInMemory("\n", resultPath);
- }
-
- @Test
- public void testDataDuplicatingMapper() throws Exception {
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
- DataSet<Tuple2<IntWritable, Text>> duplicatingFlatMapDs = ds.
- flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new DuplicatingMapper()));
-
- String resultPath = tempFolder.newFile().toURI().toString();
-
- duplicatingFlatMapDs.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
- env.execute();
-
- String expected = "(1,Hi)\n" + "(1,HI)\n" +
- "(2,Hello)\n" + "(2,HELLO)\n" +
- "(3,Hello world)\n" + "(3,HELLO WORLD)\n" +
- "(4,Hello world, how are you?)\n" + "(4,HELLO WORLD, HOW ARE YOU?)\n" +
- "(5,I am fine.)\n" + "(5,I AM FINE.)\n" +
- "(6,Luke Skywalker)\n" + "(6,LUKE SKYWALKER)\n" +
- "(7,Comment#1)\n" + "(7,COMMENT#1)\n" +
- "(8,Comment#2)\n" + "(8,COMMENT#2)\n" +
- "(9,Comment#3)\n" + "(9,COMMENT#3)\n" +
- "(10,Comment#4)\n" + "(10,COMMENT#4)\n" +
- "(11,Comment#5)\n" + "(11,COMMENT#5)\n" +
- "(12,Comment#6)\n" + "(12,COMMENT#6)\n" +
- "(13,Comment#7)\n" + "(13,COMMENT#7)\n" +
- "(14,Comment#8)\n" + "(14,COMMENT#8)\n" +
- "(15,Comment#9)\n" + "(15,COMMENT#9)\n" +
- "(16,Comment#10)\n" + "(16,COMMENT#10)\n" +
- "(17,Comment#11)\n" + "(17,COMMENT#11)\n" +
- "(18,Comment#12)\n" + "(18,COMMENT#12)\n" +
- "(19,Comment#13)\n" + "(19,COMMENT#13)\n" +
- "(20,Comment#14)\n" + "(20,COMMENT#14)\n" +
- "(21,Comment#15)\n" + "(21,COMMENT#15)\n";
-
- compareResultsByLinesInMemory(expected, resultPath);
- }
-
- @Test
- public void testConfigurableMapper() throws Exception {
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- JobConf conf = new JobConf();
- conf.set("my.filterPrefix", "Hello");
-
- DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
- DataSet<Tuple2<IntWritable, Text>> hellos = ds.
- flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new ConfigurableMapper(), conf));
-
- String resultPath = tempFolder.newFile().toURI().toString();
-
- hellos.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
- env.execute();
-
- String expected = "(2,Hello)\n" +
- "(3,Hello world)\n" +
- "(4,Hello world, how are you?)\n";
-
- compareResultsByLinesInMemory(expected, resultPath);
- }
-
-
-
- public static class NonPassingMapper implements Mapper<IntWritable, Text, IntWritable, Text> {
-
- @Override
- public void map(final IntWritable k, final Text v,
- final OutputCollector<IntWritable, Text> out, final Reporter r) throws IOException {
- if ( v.toString().contains("bananas") ) {
- out.collect(k,v);
- }
- }
-
- @Override
- public void configure(final JobConf arg0) { }
-
- @Override
- public void close() throws IOException { }
- }
-
- public static class DuplicatingMapper implements Mapper<IntWritable, Text, IntWritable, Text> {
-
- @Override
- public void map(final IntWritable k, final Text v,
- final OutputCollector<IntWritable, Text> out, final Reporter r) throws IOException {
- out.collect(k, v);
- out.collect(k, new Text(v.toString().toUpperCase()));
- }
-
- @Override
- public void configure(final JobConf arg0) { }
-
- @Override
- public void close() throws IOException { }
- }
-
- public static class ConfigurableMapper implements Mapper<IntWritable, Text, IntWritable, Text> {
- private String filterPrefix;
-
- @Override
- public void map(IntWritable k, Text v, OutputCollector<IntWritable, Text> out, Reporter r)
- throws IOException {
- if(v.toString().startsWith(filterPrefix)) {
- out.collect(k, v);
- }
- }
-
- @Override
- public void configure(JobConf c) {
- filterPrefix = c.get("my.filterPrefix");
- }
-
- @Override
- public void close() throws IOException { }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapredITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapredITCase.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapredITCase.java
deleted file mode 100644
index b6650d2..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopMapredITCase.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.hadoopcompatibility.mapred;
-
-import org.apache.flink.hadoopcompatibility.mapred.example.HadoopMapredCompatWordCount;
-import org.apache.flink.test.testdata.WordCountData;
-import org.apache.flink.test.util.JavaProgramTestBase;
-
-public class HadoopMapredITCase extends JavaProgramTestBase {
-
- protected String textPath;
- protected String resultPath;
-
- @Override
- protected void preSubmit() throws Exception {
- textPath = createTempFile("text.txt", WordCountData.TEXT);
- resultPath = getTempDirPath("result");
- this.setDegreeOfParallelism(4);
- }
-
- @Override
- protected void postSubmit() throws Exception {
- compareResultsByLinesInMemory(WordCountData.COUNTS, resultPath, new String[]{".", "_"});
- }
-
- @Override
- protected void testProgram() throws Exception {
- HadoopMapredCompatWordCount.main(new String[] { textPath, resultPath });
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceCombineFunctionITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceCombineFunctionITCase.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceCombineFunctionITCase.java
deleted file mode 100644
index 92b0dc3..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceCombineFunctionITCase.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.hadoopcompatibility.mapred;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.hadoopcompatibility.mapred.HadoopReduceCombineFunction;
-import org.apache.flink.hadoopcompatibility.mapred.HadoopReduceFunction;
-import org.apache.flink.test.util.MultipleProgramsTestBase;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.hamcrest.core.IsEqual;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class HadoopReduceCombineFunctionITCase extends MultipleProgramsTestBase {
-
- public HadoopReduceCombineFunctionITCase(ExecutionMode mode){
- super(mode);
- }
-
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
-
- @Test
- public void testStandardCountingWithCombiner() throws Exception{
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<IntWritable, IntWritable>> ds = HadoopTestData.getKVPairDataSet(env).
- map(new Mapper1());
-
- DataSet<Tuple2<IntWritable, IntWritable>> counts = ds.
- groupBy(0).
- reduceGroup(new HadoopReduceCombineFunction<IntWritable, IntWritable, IntWritable, IntWritable>(
- new SumReducer(), new SumReducer()));
-
- String resultPath = tempFolder.newFile().toURI().toString();
-
- counts.writeAsText(resultPath);
- env.execute();
-
- String expected = "(0,5)\n"+
- "(1,6)\n" +
- "(2,6)\n" +
- "(3,4)\n";
-
- compareResultsByLinesInMemory(expected, resultPath);
- }
-
- @Test
- public void testUngroupedHadoopReducer() throws Exception {
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<IntWritable, IntWritable>> ds = HadoopTestData.getKVPairDataSet(env).
- map(new Mapper2());
-
- DataSet<Tuple2<IntWritable, IntWritable>> sum = ds.
- reduceGroup(new HadoopReduceCombineFunction<IntWritable, IntWritable, IntWritable, IntWritable>(
- new SumReducer(), new SumReducer()));
-
- String resultPath = tempFolder.newFile().toURI().toString();
-
- sum.writeAsText(resultPath);
- env.execute();
-
- String expected = "(0,231)\n";
-
- compareResultsByLinesInMemory(expected, resultPath);
- }
-
- @Test
- public void testCombiner() throws Exception {
- org.junit.Assume.assumeThat(mode, new IsEqual<ExecutionMode>(ExecutionMode.CLUSTER));
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<IntWritable, IntWritable>> ds = HadoopTestData.getKVPairDataSet(env).
- map(new Mapper3());
-
- DataSet<Tuple2<IntWritable, IntWritable>> counts = ds.
- groupBy(0).
- reduceGroup(new HadoopReduceCombineFunction<IntWritable, IntWritable, IntWritable, IntWritable>(
- new SumReducer(), new KeyChangingReducer()));
-
- String resultPath = tempFolder.newFile().toURI().toString();
-
- counts.writeAsText(resultPath);
- env.execute();
-
- String expected = "(0,5)\n"+
- "(1,6)\n" +
- "(2,5)\n" +
- "(3,5)\n";
-
- compareResultsByLinesInMemory(expected, resultPath);
- }
-
- @Test
- public void testConfigurationViaJobConf() throws Exception {
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- JobConf conf = new JobConf();
- conf.set("my.cntPrefix", "Hello");
-
- DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env).
- map(new Mapper4());
-
- DataSet<Tuple2<IntWritable, IntWritable>> hellos = ds.
- groupBy(0).
- reduceGroup(new HadoopReduceFunction<IntWritable, Text, IntWritable, IntWritable>(
- new ConfigurableCntReducer(), conf));
-
- String resultPath = tempFolder.newFile().toURI().toString();
-
- hellos.writeAsText(resultPath);
- env.execute();
-
- // return expected result
- String expected = "(0,0)\n"+
- "(1,0)\n" +
- "(2,1)\n" +
- "(3,1)\n" +
- "(4,1)\n";
-
- compareResultsByLinesInMemory(expected, resultPath);
- }
-
- public static class SumReducer implements Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
-
- @Override
- public void reduce(IntWritable k, Iterator<IntWritable> v, OutputCollector<IntWritable, IntWritable> out, Reporter r)
- throws IOException {
-
- int sum = 0;
- while(v.hasNext()) {
- sum += v.next().get();
- }
- out.collect(k, new IntWritable(sum));
- }
-
- @Override
- public void configure(JobConf arg0) { }
-
- @Override
- public void close() throws IOException { }
- }
-
- public static class KeyChangingReducer implements Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
-
- @Override
- public void reduce(IntWritable k, Iterator<IntWritable> v, OutputCollector<IntWritable, IntWritable> out, Reporter r)
- throws IOException {
- while(v.hasNext()) {
- out.collect(new IntWritable(k.get() % 4), v.next());
- }
- }
-
- @Override
- public void configure(JobConf arg0) { }
-
- @Override
- public void close() throws IOException { }
- }
-
- public static class ConfigurableCntReducer implements Reducer<IntWritable, Text, IntWritable, IntWritable> {
- private String countPrefix;
-
- @Override
- public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r)
- throws IOException {
- int commentCnt = 0;
- while(vs.hasNext()) {
- String v = vs.next().toString();
- if(v.startsWith(this.countPrefix)) {
- commentCnt++;
- }
- }
- out.collect(k, new IntWritable(commentCnt));
- }
-
- @Override
- public void configure(final JobConf c) {
- this.countPrefix = c.get("my.cntPrefix");
- }
-
- @Override
- public void close() throws IOException { }
- }
-
- public static class Mapper1 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable,
- IntWritable>> {
- private static final long serialVersionUID = 1L;
- Tuple2<IntWritable,IntWritable> outT = new Tuple2<IntWritable,IntWritable>();
- @Override
- public Tuple2<IntWritable, IntWritable> map(Tuple2<IntWritable, Text> v)
- throws Exception {
- outT.f0 = new IntWritable(v.f0.get() / 6);
- outT.f1 = new IntWritable(1);
- return outT;
- }
- }
-
- public static class Mapper2 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable,
- IntWritable>> {
- private static final long serialVersionUID = 1L;
- Tuple2<IntWritable,IntWritable> outT = new Tuple2<IntWritable,IntWritable>();
- @Override
- public Tuple2<IntWritable, IntWritable> map(Tuple2<IntWritable, Text> v)
- throws Exception {
- outT.f0 = new IntWritable(0);
- outT.f1 = v.f0;
- return outT;
- }
- }
-
- public static class Mapper3 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable, IntWritable>> {
- private static final long serialVersionUID = 1L;
- Tuple2<IntWritable,IntWritable> outT = new Tuple2<IntWritable,IntWritable>();
- @Override
- public Tuple2<IntWritable, IntWritable> map(Tuple2<IntWritable, Text> v)
- throws Exception {
- outT.f0 = v.f0;
- outT.f1 = new IntWritable(1);
- return outT;
- }
- }
-
- public static class Mapper4 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable, Text>> {
- private static final long serialVersionUID = 1L;
- @Override
- public Tuple2<IntWritable, Text> map(Tuple2<IntWritable, Text> v)
- throws Exception {
- v.f0 = new IntWritable(v.f0.get() % 5);
- return v;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceFunctionITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceFunctionITCase.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceFunctionITCase.java
deleted file mode 100644
index 1801c3e..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopReduceFunctionITCase.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.hadoopcompatibility.mapred;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.hadoopcompatibility.mapred.HadoopReduceFunction;
-import org.apache.flink.test.util.MultipleProgramsTestBase;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class HadoopReduceFunctionITCase extends MultipleProgramsTestBase {
-
- public HadoopReduceFunctionITCase(ExecutionMode mode){
- super(mode);
- }
-
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
-
- @Test
- public void testStandardGrouping() throws Exception{
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env).
- map(new Mapper1());
-
- DataSet<Tuple2<IntWritable, IntWritable>> commentCnts = ds.
- groupBy(0).
- reduceGroup(new HadoopReduceFunction<IntWritable, Text, IntWritable, IntWritable>(new CommentCntReducer()));
-
- String resultPath = tempFolder.newFile().toURI().toString();
-
- commentCnts.writeAsText(resultPath);
- env.execute();
-
- String expected = "(0,0)\n"+
- "(1,3)\n" +
- "(2,5)\n" +
- "(3,5)\n" +
- "(4,2)\n";
-
- compareResultsByLinesInMemory(expected, resultPath);
- }
-
- @Test
- public void testUngroupedHadoopReducer() throws Exception {
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
-
- DataSet<Tuple2<IntWritable, IntWritable>> commentCnts = ds.
- reduceGroup(new HadoopReduceFunction<IntWritable, Text, IntWritable, IntWritable>(new AllCommentCntReducer()));
-
- String resultPath = tempFolder.newFile().toURI().toString();
-
- commentCnts.writeAsText(resultPath);
- env.execute();
-
- String expected = "(42,15)\n";
-
- compareResultsByLinesInMemory(expected, resultPath);
- }
-
- @Test
- public void testConfigurationViaJobConf() throws Exception {
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- JobConf conf = new JobConf();
- conf.set("my.cntPrefix", "Hello");
-
- DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env).
- map(new Mapper2());
-
- DataSet<Tuple2<IntWritable, IntWritable>> helloCnts = ds.
- groupBy(0).
- reduceGroup(new HadoopReduceFunction<IntWritable, Text, IntWritable, IntWritable>(
- new ConfigurableCntReducer(), conf));
-
- String resultPath = tempFolder.newFile().toURI().toString();
-
- helloCnts.writeAsText(resultPath);
- env.execute();
-
- String expected = "(0,0)\n"+
- "(1,0)\n" +
- "(2,1)\n" +
- "(3,1)\n" +
- "(4,1)\n";
-
- compareResultsByLinesInMemory(expected, resultPath);
- }
-
- public static class CommentCntReducer implements Reducer<IntWritable, Text, IntWritable, IntWritable> {
-
- @Override
- public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r)
- throws IOException {
- int commentCnt = 0;
- while(vs.hasNext()) {
- String v = vs.next().toString();
- if(v.startsWith("Comment")) {
- commentCnt++;
- }
- }
- out.collect(k, new IntWritable(commentCnt));
- }
-
- @Override
- public void configure(final JobConf arg0) { }
-
- @Override
- public void close() throws IOException { }
- }
-
- public static class AllCommentCntReducer implements Reducer<IntWritable, Text, IntWritable, IntWritable> {
-
- @Override
- public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r)
- throws IOException {
- int commentCnt = 0;
- while(vs.hasNext()) {
- String v = vs.next().toString();
- if(v.startsWith("Comment")) {
- commentCnt++;
- }
- }
- out.collect(new IntWritable(42), new IntWritable(commentCnt));
- }
-
- @Override
- public void configure(final JobConf arg0) { }
-
- @Override
- public void close() throws IOException { }
- }
-
- public static class ConfigurableCntReducer implements Reducer<IntWritable, Text, IntWritable, IntWritable> {
- private String countPrefix;
-
- @Override
- public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r)
- throws IOException {
- int commentCnt = 0;
- while(vs.hasNext()) {
- String v = vs.next().toString();
- if(v.startsWith(this.countPrefix)) {
- commentCnt++;
- }
- }
- out.collect(k, new IntWritable(commentCnt));
- }
-
- @Override
- public void configure(final JobConf c) {
- this.countPrefix = c.get("my.cntPrefix");
- }
-
- @Override
- public void close() throws IOException { }
- }
-
- public static class Mapper1 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable, Text>> {
- private static final long serialVersionUID = 1L;
- @Override
- public Tuple2<IntWritable, Text> map(Tuple2<IntWritable, Text> v)
- throws Exception {
- v.f0 = new IntWritable(v.f0.get() / 5);
- return v;
- }
- }
-
- public static class Mapper2 implements MapFunction<Tuple2<IntWritable, Text>, Tuple2<IntWritable, Text>> {
- private static final long serialVersionUID = 1L;
- @Override
- public Tuple2<IntWritable, Text> map(Tuple2<IntWritable, Text> v)
- throws Exception {
- v.f0 = new IntWritable(v.f0.get() % 5);
- return v;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopTestData.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopTestData.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopTestData.java
deleted file mode 100644
index eed6f8f..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/HadoopTestData.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.hadoopcompatibility.mapred;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-
-public class HadoopTestData {
-
- public static DataSet<Tuple2<IntWritable, Text>> getKVPairDataSet(ExecutionEnvironment env) {
-
- List<Tuple2<IntWritable, Text>> data = new ArrayList<Tuple2<IntWritable, Text>>();
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(1),new Text("Hi")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(2),new Text("Hello")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(3),new Text("Hello world")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(4),new Text("Hello world, how are you?")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(5),new Text("I am fine.")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(6),new Text("Luke Skywalker")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(7),new Text("Comment#1")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(8),new Text("Comment#2")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(9),new Text("Comment#3")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(10),new Text("Comment#4")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(11),new Text("Comment#5")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(12),new Text("Comment#6")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(13),new Text("Comment#7")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(14),new Text("Comment#8")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(15),new Text("Comment#9")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(16),new Text("Comment#10")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(17),new Text("Comment#11")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(18),new Text("Comment#12")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(19),new Text("Comment#13")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(20),new Text("Comment#14")));
- data.add(new Tuple2<IntWritable, Text>(new IntWritable(21),new Text("Comment#15")));
-
- Collections.shuffle(data);
-
- return env.fromCollection(data);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/record/HadoopRecordInputOutputITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/record/HadoopRecordInputOutputITCase.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/record/HadoopRecordInputOutputITCase.java
deleted file mode 100644
index fe7ea8e..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/record/HadoopRecordInputOutputITCase.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.test.hadoopcompatibility.mapred.record;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.hadoopcompatibility.mapred.record.example.WordCountWithOutputFormat;
-import org.apache.flink.test.testdata.WordCountData;
-import org.apache.flink.test.util.RecordAPITestBase;
-
-/**
- * test the hadoop inputformat and outputformat
- */
-public class HadoopRecordInputOutputITCase extends RecordAPITestBase {
- protected String textPath;
- protected String resultPath;
- protected String counts;
-
- @Override
- protected void preSubmit() throws Exception {
- textPath = createTempFile("text.txt", WordCountData.TEXT);
- resultPath = getTempDirPath("result");
- counts = WordCountData.COUNTS.replaceAll(" ", "\t");
- }
-
- @Override
- protected Plan getTestJob() {
- //WordCountWithHadoopOutputFormat takes hadoop TextInputFormat as input and output file in hadoop TextOutputFormat
- WordCountWithOutputFormat wc = new WordCountWithOutputFormat();
- return wc.getPlan("1", textPath, resultPath);
- }
-
- @Override
- protected void postSubmit() throws Exception {
- // Test results, append /1 to resultPath due to the generated _temproray file.
- compareResultsByLinesInMemory(counts, resultPath + "/1");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIteratorTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIteratorTest.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIteratorTest.java
deleted file mode 100644
index 2592b88..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapred/wrapper/HadoopTupleUnwrappingIteratorTest.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.hadoopcompatibility.mapred.wrapper;
-
-import java.util.ArrayList;
-import java.util.NoSuchElementException;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopTupleUnwrappingIterator;
-import org.apache.hadoop.io.IntWritable;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class HadoopTupleUnwrappingIteratorTest {
-
- @Test
- public void testValueIterator() {
-
- HadoopTupleUnwrappingIterator<IntWritable, IntWritable> valIt =
- new HadoopTupleUnwrappingIterator<IntWritable, IntWritable>(IntWritable.class);
-
- // many values
-
- ArrayList<Tuple2<IntWritable, IntWritable>> tList = new ArrayList<Tuple2<IntWritable, IntWritable>>();
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(1)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(2)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(3)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(4)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(5)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(6)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(7)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(1),new IntWritable(8)));
-
- int expectedKey = 1;
- int[] expectedValues = new int[] {1,2,3,4,5,6,7,8};
-
- valIt.set(tList.iterator());
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
- for(int expectedValue : expectedValues) {
- Assert.assertTrue(valIt.hasNext());
- Assert.assertTrue(valIt.hasNext());
- Assert.assertTrue(valIt.next().get() == expectedValue);
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
- }
- Assert.assertFalse(valIt.hasNext());
- Assert.assertFalse(valIt.hasNext());
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
-
- // one value
-
- tList.clear();
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(2),new IntWritable(10)));
-
- expectedKey = 2;
- expectedValues = new int[]{10};
-
- valIt.set(tList.iterator());
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
- for(int expectedValue : expectedValues) {
- Assert.assertTrue(valIt.hasNext());
- Assert.assertTrue(valIt.hasNext());
- Assert.assertTrue(valIt.next().get() == expectedValue);
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
- }
- Assert.assertFalse(valIt.hasNext());
- Assert.assertFalse(valIt.hasNext());
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
-
- // more values
-
- tList.clear();
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(3),new IntWritable(10)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(3),new IntWritable(4)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(3),new IntWritable(7)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(3),new IntWritable(9)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(21)));
-
- expectedKey = 3;
- expectedValues = new int[]{10,4,7,9,21};
-
- valIt.set(tList.iterator());
- Assert.assertTrue(valIt.hasNext());
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
- for(int expectedValue : expectedValues) {
- Assert.assertTrue(valIt.hasNext());
- Assert.assertTrue(valIt.hasNext());
- Assert.assertTrue(valIt.next().get() == expectedValue);
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
- }
- Assert.assertFalse(valIt.hasNext());
- Assert.assertFalse(valIt.hasNext());
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
-
- // no has next calls
-
- tList.clear();
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(5)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(8)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(42)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(-1)));
- tList.add(new Tuple2<IntWritable, IntWritable>(new IntWritable(4),new IntWritable(0)));
-
- expectedKey = 4;
- expectedValues = new int[]{5,8,42,-1,0};
-
- valIt.set(tList.iterator());
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
- for(int expectedValue : expectedValues) {
- Assert.assertTrue(valIt.next().get() == expectedValue);
- }
- try {
- valIt.next();
- Assert.fail();
- } catch (NoSuchElementException nsee) {
- // expected
- }
- Assert.assertFalse(valIt.hasNext());
- Assert.assertTrue(valIt.getCurrentKey().get() == expectedKey);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputFormatTest.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputFormatTest.java
deleted file mode 100644
index d79afaa..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputFormatTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.test.hadoopcompatibility.mapreduce;
-
-
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.hadoopcompatibility.mapreduce.HadoopInputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.junit.Assert.fail;
-
-
-
-public class HadoopInputFormatTest {
-
-
- public class DummyVoidKeyInputFormat<T> extends FileInputFormat<Void, T> {
-
- public DummyVoidKeyInputFormat() {
- }
-
- @Override
- public RecordReader<Void, T> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
- return null;
- }
- }
-
-
- @Test
- public void checkTypeInformation() {
- try {
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- // Set up the Hadoop Input Format
- Job job = Job.getInstance();
- HadoopInputFormat<Void, Long> hadoopInputFormat = new HadoopInputFormat<Void, Long>( new DummyVoidKeyInputFormat(), Void.class, Long.class, job);
-
- TypeInformation<Tuple2<Void,Long>> tupleType = hadoopInputFormat.getProducedType();
- TypeInformation<Tuple2<Void,Long>> testTupleType = new TupleTypeInfo<Tuple2<Void,Long>>(BasicTypeInfo.VOID_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
-
- if(tupleType.isTupleType()) {
- if(!((TupleTypeInfo)tupleType).equals(testTupleType)) {
- fail("Tuple type information was not set correctly!");
- }
- } else {
- fail("Type information was not set to tuple type information!");
- }
-
- }
- catch (Exception ex) {
- fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputOutputITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputOutputITCase.java b/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputOutputITCase.java
deleted file mode 100644
index 7eee629..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/HadoopInputOutputITCase.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.hadoopcompatibility.mapreduce;
-
-import org.apache.flink.hadoopcompatibility.mapreduce.example.WordCount;
-import org.apache.flink.test.testdata.WordCountData;
-import org.apache.flink.test.util.JavaProgramTestBase;
-
-public class HadoopInputOutputITCase extends JavaProgramTestBase {
-
- protected String textPath;
- protected String resultPath;
-
-
- @Override
- protected void preSubmit() throws Exception {
- textPath = createTempFile("text.txt", WordCountData.TEXT);
- resultPath = getTempDirPath("result");
- this.setDegreeOfParallelism(4);
- }
-
- @Override
- protected void postSubmit() throws Exception {
- compareResultsByLinesInMemory(WordCountData.COUNTS, resultPath, new String[]{".", "_"});
- }
-
- @Override
- protected void testProgram() throws Exception {
- WordCount.main(new String[] { textPath, resultPath });
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/resources/log4j-test.properties b/flink-addons/flink-hadoop-compatibility/src/test/resources/log4j-test.properties
deleted file mode 100644
index 0b686e5..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/resources/log4j-test.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# Set root logger level to DEBUG and its only appender to A1.
-log4j.rootLogger=OFF, A1
-
-# A1 is set to be a ConsoleAppender.
-log4j.appender.A1=org.apache.log4j.ConsoleAppender
-
-# A1 uses PatternLayout.
-log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hadoop-compatibility/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/test/resources/logback-test.xml b/flink-addons/flink-hadoop-compatibility/src/test/resources/logback-test.xml
deleted file mode 100644
index 8b3bb27..0000000
--- a/flink-addons/flink-hadoop-compatibility/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
- </encoder>
- </appender>
-
- <root level="WARN">
- <appender-ref ref="STDOUT"/>
- </root>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hbase/pom.xml b/flink-addons/flink-hbase/pom.xml
deleted file mode 100644
index 6807b0c..0000000
--- a/flink-addons/flink-hbase/pom.xml
+++ /dev/null
@@ -1,165 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-addons</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-hbase</artifactId>
- <name>flink-hbase</name>
- <packaging>jar</packaging>
-
- <properties>
- <hbase.hadoop1.version>0.98.6.1-hadoop1</hbase.hadoop1.version>
- <hbase.hadoop2.version>0.98.6.1-hadoop2</hbase.hadoop2.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-hadoop-compatibility</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${hbase.version}</version>
- <exclusions>
- <!-- Remove unneeded dependency, which is conflicting with our jetty-util version. -->
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
- <profiles>
- <profile>
- <id>hadoop-1</id>
- <activation>
- <property>
- <!-- Please do not remove the 'hadoop1' comment. See ./tools/generate_specific_pom.sh -->
- <!--hadoop1--><name>hadoop.profile</name><value>1</value>
- </property>
- </activation>
- <properties>
- <hbase.version>${hbase.hadoop1.version}</hbase.version>
- </properties>
- <dependencies>
- <!-- Force hadoop-common dependency -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>hadoop-2</id>
- <activation>
- <property>
- <!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh -->
- <!--hadoop2--><name>!hadoop.profile</name>
- </property>
- </activation>
- <properties>
- <hbase.version>${hbase.hadoop2.version}</hbase.version>
- </properties>
- <dependencies>
- <!-- Force hadoop-common dependency -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>cdh5.1.3</id>
- <properties>
- <hadoop.profile>2</hadoop.profile>
- <hbase.version>0.98.1-cdh5.1.3</hbase.version>
- <hadoop.version>2.3.0-cdh5.1.3</hadoop.version>
- <!-- Cloudera use different versions for hadoop core and commons-->
- <!-- This profile could be removed if Cloudera fix this mismatch! -->
- <hadoop.core.version>2.3.0-mr1-cdh5.1.3</hadoop.core.version>
- </properties>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>${hadoop.core.version}</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
- <dependencies>
- <!-- Force hadoop-common dependency -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
- </profiles>
-
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputFormat.java b/flink-addons/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputFormat.java
deleted file mode 100755
index 9c861ed..0000000
--- a/flink-addons/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputFormat.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.addons.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.io.LocatableInputSplitAssigner;
-import org.apache.flink.api.common.io.statistics.BaseStatistics;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.io.InputSplitAssigner;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * {@link InputFormat} subclass that wraps the access for HTables.
- *
- */
-public abstract class TableInputFormat<T extends Tuple> implements InputFormat<T, TableInputSplit>{
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(TableInputFormat.class);
-
- /** helper variable to decide whether the input is exhausted or not */
- private boolean endReached = false;
-
- // TODO table and scan could be serialized when kryo serializer will be the default
- protected transient HTable table;
- protected transient Scan scan;
-
- /** HBase iterator wrapper */
- private ResultScanner rs;
-
- private byte[] lastRow;
- private int scannedRows;
-
- // abstract methods allow for multiple table and scanners in the same job
- protected abstract Scan getScanner();
- protected abstract String getTableName();
- protected abstract T mapResultToTuple(Result r);
-
- /**
- * creates a {@link Scan} object and a {@link HTable} connection
- *
- * @param parameters
- * @see Configuration
- */
- @Override
- public void configure(Configuration parameters) {
- this.table = createTable();
- this.scan = getScanner();
- }
-
- /** Create an {@link HTable} instance and set it into this format */
- private HTable createTable() {
- LOG.info("Initializing HBaseConfiguration");
- //use files found in the classpath
- org.apache.hadoop.conf.Configuration hConf = HBaseConfiguration.create();
-
- try {
- return new HTable(hConf, getTableName());
- } catch (Exception e) {
- LOG.error("Error instantiating a new HTable instance", e);
- }
- return null;
- }
-
- @Override
- public boolean reachedEnd() throws IOException {
- return this.endReached;
- }
-
- @Override
- public T nextRecord(T reuse) throws IOException {
- if (this.rs == null){
- throw new IOException("No table result scanner provided!");
- }
- try{
- Result res = this.rs.next();
- if (res != null){
- scannedRows++;
- lastRow = res.getRow();
- return mapResultToTuple(res);
- }
- }catch (Exception e) {
- this.rs.close();
- //workaround for timeout on scan
- StringBuffer logMsg = new StringBuffer("Error after scan of ")
- .append(scannedRows)
- .append(" rows. Retry with a new scanner...");
- LOG.warn(logMsg.toString(), e);
- this.scan.setStartRow(lastRow);
- this.rs = table.getScanner(scan);
- Result res = this.rs.next();
- if (res != null) {
- scannedRows++;
- lastRow = res.getRow();
- return mapResultToTuple(res);
- }
- }
-
- this.endReached = true;
- return null;
- }
-
- @Override
- public void open(TableInputSplit split) throws IOException {
- if (split == null){
- throw new IOException("Input split is null!");
- }
- if (table == null){
- throw new IOException("No HTable provided!");
- }
- if (scan == null){
- throw new IOException("No Scan instance provided");
- }
-
- logSplitInfo("opening", split);
- scan.setStartRow(split.getStartRow());
- lastRow = split.getEndRow();
- scan.setStopRow(lastRow);
-
- this.rs = table.getScanner(scan);
- this.endReached = false;
- this.scannedRows = 0;
- }
-
- @Override
- public void close() throws IOException {
- if(rs!=null){
- this.rs.close();
- }
- if(table!=null){
- this.table.close();
- }
- LOG.info("Closing split (scanned {} rows)", scannedRows);
- this.lastRow = null;
- }
-
- @Override
- public TableInputSplit[] createInputSplits(final int minNumSplits) throws IOException {
- //Gets the starting and ending row keys for every region in the currently open table
- final Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
- if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) {
- throw new IOException("Expecting at least one region.");
- }
- final byte[] startRow = scan.getStartRow();
- final byte[] stopRow = scan.getStopRow();
- final boolean scanWithNoLowerBound = startRow.length == 0;
- final boolean scanWithNoUpperBound = stopRow.length == 0;
-
- final List<TableInputSplit> splits = new ArrayList<TableInputSplit>(minNumSplits);
- for (int i = 0; i < keys.getFirst().length; i++) {
- final byte[] startKey = keys.getFirst()[i];
- final byte[] endKey = keys.getSecond()[i];
- final String regionLocation = table.getRegionLocation(startKey, false).getHostnamePort();
- //Test if the given region is to be included in the InputSplit while splitting the regions of a table
- if (!includeRegionInSplit(startKey, endKey)) {
- continue;
- }
- //Finds the region on which the given row is being served
- final String[] hosts = new String[] { regionLocation };
-
- // determine if regions contains keys used by the scan
- boolean isLastRegion = endKey.length == 0;
- if ((scanWithNoLowerBound || isLastRegion || Bytes.compareTo(startRow, endKey) < 0) &&
- (scanWithNoUpperBound || Bytes.compareTo(stopRow, startKey) > 0)) {
-
- final byte[] splitStart = scanWithNoLowerBound || Bytes.compareTo(startKey, startRow) >= 0 ? startKey : startRow;
- final byte[] splitStop = (scanWithNoUpperBound || Bytes.compareTo(endKey, stopRow) <= 0)
- && !isLastRegion ? endKey : stopRow;
- int id = splits.size();
- final TableInputSplit split = new TableInputSplit(id, hosts, table.getTableName(), splitStart, splitStop);
- splits.add(split);
- }
- }
- LOG.info("Created " + splits.size() + " splits");
- for (TableInputSplit split : splits) {
- logSplitInfo("created", split);
- }
- return splits.toArray(new TableInputSplit[0]);
- }
-
- private void logSplitInfo(String action, TableInputSplit split) {
- int splitId = split.getSplitNumber();
- String splitStart = Bytes.toString(split.getStartRow());
- String splitEnd = Bytes.toString(split.getEndRow());
- String splitStartKey = splitStart.isEmpty() ? "-" : splitStart;
- String splitStopKey = splitEnd.isEmpty() ? "-" : splitEnd;
- String[] hostnames = split.getHostnames();
- LOG.info("{} split [{}|{}|{}|{}]",action, splitId, hostnames, splitStartKey, splitStopKey);
- }
-
- /**
- * Test if the given region is to be included in the InputSplit while splitting the regions of a table.
- * <p>
- * This optimization is effective when there is a specific reasoning to exclude an entire region from the M-R job,
- * (and hence, not contributing to the InputSplit), given the start and end keys of the same. <br>
- * Useful when we need to remember the last-processed top record and revisit the [last, current) interval for M-R
- * processing, continuously. In addition to reducing InputSplits, reduces the load on the region server as well, due
- * to the ordering of the keys. <br>
- * <br>
- * Note: It is possible that <code>endKey.length() == 0 </code> , for the last (recent) region. <br>
- * Override this method, if you want to bulk exclude regions altogether from M-R. By default, no region is excluded(
- * i.e. all regions are included).
- *
- * @param startKey
- * Start key of the region
- * @param endKey
- * End key of the region
- * @return true, if this region needs to be included as part of the input (default).
- */
- private static boolean includeRegionInSplit(final byte[] startKey, final byte[] endKey) {
- return true;
- }
-
- @Override
- public InputSplitAssigner getInputSplitAssigner(TableInputSplit[] inputSplits) {
- return new LocatableInputSplitAssigner(inputSplits);
- }
-
- @Override
- public BaseStatistics getStatistics(BaseStatistics cachedStatistics) {
- return null;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputSplit.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputSplit.java b/flink-addons/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputSplit.java
deleted file mode 100644
index 6d8bf42..0000000
--- a/flink-addons/flink-hbase/src/main/java/org/apache/flink/addons/hbase/TableInputSplit.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.addons.hbase;
-
-import java.io.IOException;
-
-import org.apache.flink.core.io.LocatableInputSplit;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-
-/**
- * This class implements a input splits for HBase. Each table input split corresponds to a key range (low, high). All
- * references to row below refer to the key of the row.
- */
-public class TableInputSplit extends LocatableInputSplit {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * The name of the table to retrieve data from
- */
- private byte[] tableName;
-
- /**
- * The start row of the split.
- */
- private byte[] startRow;
-
- /**
- * The end row of the split.
- */
- private byte[] endRow;
-
- /**
- * Creates a new table input split
- *
- * @param splitNumber
- * the number of the input split
- * @param hostnames
- * the names of the hosts storing the data the input split refers to
- * @param tableName
- * the name of the table to retrieve data from
- * @param startRow
- * the start row of the split
- * @param endRow
- * the end row of the split
- */
- TableInputSplit(final int splitNumber, final String[] hostnames, final byte[] tableName, final byte[] startRow,
- final byte[] endRow) {
- super(splitNumber, hostnames);
-
- this.tableName = tableName;
- this.startRow = startRow;
- this.endRow = endRow;
- }
-
- /**
- * Default constructor for serialization/deserialization.
- */
- public TableInputSplit() {
- super();
-
- this.tableName = null;
- this.startRow = null;
- this.endRow = null;
- }
-
- /**
- * Returns the table name.
- *
- * @return The table name.
- */
- public byte[] getTableName() {
- return this.tableName;
- }
-
- /**
- * Returns the start row.
- *
- * @return The start row.
- */
- public byte[] getStartRow() {
- return this.startRow;
- }
-
- /**
- * Returns the end row.
- *
- * @return The end row.
- */
- public byte[] getEndRow() {
- return this.endRow;
- }
-
-
- @Override
- public void write(final DataOutputView out) throws IOException {
-
- super.write(out);
-
- // Write the table name
- if (this.tableName == null) {
- out.writeInt(-1);
- } else {
- out.writeInt(this.tableName.length);
- out.write(this.tableName);
- }
-
- // Write the start row
- if (this.startRow == null) {
- out.writeInt(-1);
- } else {
- out.writeInt(this.startRow.length);
- out.write(this.startRow);
- }
-
- // Write the end row
- if (this.endRow == null) {
- out.writeInt(-1);
- } else {
- out.writeInt(this.endRow.length);
- out.write(this.endRow);
- }
- }
-
-
- @Override
- public void read(final DataInputView in) throws IOException {
-
- super.read(in);
-
- // Read the table name
- int len = in.readInt();
- if (len >= 0) {
- this.tableName = new byte[len];
- in.readFully(this.tableName);
- }
-
- // Read the start row
- len = in.readInt();
- if (len >= 0) {
- this.startRow = new byte[len];
- in.readFully(this.startRow);
- }
-
- // Read the end row
- len = in.readInt();
- if (len >= 0) {
- this.endRow = new byte[len];
- in.readFully(this.endRow);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java b/flink-addons/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java
deleted file mode 100755
index b6f345a..0000000
--- a/flink-addons/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.addons.hbase.example;
-
-import org.apache.flink.addons.hbase.TableInputFormat;
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Simple stub for HBase DataSet
- *
- * To run the test first create the test table with hbase shell.
- *
- * Use the following commands:
- * <ul>
- * <li>create 'test-table', 'someCf'</li>
- * <li>put 'test-table', '1', 'someCf:someQual', 'someString'</li>
- * <li>put 'test-table', '2', 'someCf:someQual', 'anotherString'</li>
- * </ul>
- *
- * The test should return just the first entry.
- *
- */
-public class HBaseReadExample {
- public static void main(String[] args) throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- @SuppressWarnings("serial")
- DataSet<Tuple2<String, String>> hbaseDs = env.createInput(new TableInputFormat<Tuple2<String, String>>() {
- private final byte[] CF_SOME = "someCf".getBytes();
- private final byte[] Q_SOME = "someQual".getBytes();
- @Override
- public String getTableName() {
- return "test-table";
- }
-
- @Override
- protected Scan getScanner() {
- Scan scan = new Scan();
- scan.addColumn(CF_SOME, Q_SOME);
- return scan;
- }
-
- private Tuple2<String, String> reuse = new Tuple2<String, String>();
-
- @Override
- protected Tuple2<String, String> mapResultToTuple(Result r) {
- String key = Bytes.toString(r.getRow());
- String val = Bytes.toString(r.getValue(CF_SOME, Q_SOME));
- reuse.setField(key, 0);
- reuse.setField(val, 1);
- return reuse;
- }
- })
- .filter(new FilterFunction<Tuple2<String,String>>() {
-
- @Override
- public boolean filter(Tuple2<String, String> t) throws Exception {
- String val = t.getField(1);
- if(val.startsWith("someStr"))
- return true;
- return false;
- }
- });
-
- hbaseDs.print();
-
- // kick off execution.
- env.execute();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hbase/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hbase/src/test/resources/hbase-site.xml b/flink-addons/flink-hbase/src/test/resources/hbase-site.xml
deleted file mode 100644
index 2984063..0000000
--- a/flink-addons/flink-hbase/src/test/resources/hbase-site.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
-
- <property>
- <name>hbase.tmp.dir</name>
- <!--
- <value>/media/Dati/hbase-0.98-data</value>
- -->
- <value>/opt/hbase-0.98.6.1-hadoop2/data</value>
-
- </property>
- <property>
- <name>hbase.zookeeper.quorum</name>
- <value>localhost</value>
- </property>
- <!--
- <property>
- <name>hadoop.security.group.mapping</name>
- <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
- </property>
- -->
-</configuration>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-hbase/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hbase/src/test/resources/log4j.properties b/flink-addons/flink-hbase/src/test/resources/log4j.properties
deleted file mode 100755
index d6eb2b2..0000000
--- a/flink-addons/flink-hbase/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-log4j.rootLogger=${hadoop.root.logger}
-hadoop.root.logger=INFO,console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/pom.xml b/flink-addons/flink-jdbc/pom.xml
deleted file mode 100644
index ec172a9..0000000
--- a/flink-addons/flink-jdbc/pom.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-addons</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-jdbc</artifactId>
- <name>flink-jdbc</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.derby</groupId>
- <artifactId>derby</artifactId>
- <version>10.10.1.1</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-</project>
[06/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java
new file mode 100644
index 0000000..0379fe0
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple10;
+import org.apache.flink.api.java.tuple.Tuple11;
+import org.apache.flink.api.java.tuple.Tuple12;
+import org.apache.flink.api.java.tuple.Tuple13;
+import org.apache.flink.api.java.tuple.Tuple14;
+import org.apache.flink.api.java.tuple.Tuple15;
+import org.apache.flink.api.java.tuple.Tuple16;
+import org.apache.flink.api.java.tuple.Tuple17;
+import org.apache.flink.api.java.tuple.Tuple18;
+import org.apache.flink.api.java.tuple.Tuple19;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple20;
+import org.apache.flink.api.java.tuple.Tuple21;
+import org.apache.flink.api.java.tuple.Tuple22;
+import org.apache.flink.api.java.tuple.Tuple23;
+import org.apache.flink.api.java.tuple.Tuple24;
+import org.apache.flink.api.java.tuple.Tuple25;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple6;
+import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.tuple.Tuple9;
+import org.apache.flink.streaming.api.windowing.extractor.FieldsFromTuple;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FieldsFromTupleTest {
+
+ private double[] testDouble;
+
+ @Before
+ public void init() {
+ testDouble = new double[Tuple.MAX_ARITY];
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ testDouble[i] = i;
+ }
+ }
+
+ @Test
+ public void testUserSpecifiedOrder() throws InstantiationException, IllegalAccessException {
+ Tuple currentTuple = (Tuple) CLASSES[Tuple.MAX_ARITY - 1].newInstance();
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ currentTuple.setField(testDouble[i], i);
+ }
+
+ double[] expected = { testDouble[5], testDouble[3], testDouble[6], testDouble[7],
+ testDouble[0] };
+ arrayEqualityCheck(expected, new FieldsFromTuple(5, 3, 6, 7, 0).extract(currentTuple));
+
+ double[] expected2 = { testDouble[0], testDouble[Tuple.MAX_ARITY - 1] };
+ arrayEqualityCheck(expected2,
+ new FieldsFromTuple(0, Tuple.MAX_ARITY - 1).extract(currentTuple));
+
+ double[] expected3 = { testDouble[Tuple.MAX_ARITY - 1], testDouble[0] };
+ arrayEqualityCheck(expected3,
+ new FieldsFromTuple(Tuple.MAX_ARITY - 1, 0).extract(currentTuple));
+
+ double[] expected4 = { testDouble[13], testDouble[4], testDouble[5], testDouble[4],
+ testDouble[2], testDouble[8], testDouble[6], testDouble[2], testDouble[8],
+ testDouble[3], testDouble[5], testDouble[2], testDouble[16], testDouble[4],
+ testDouble[3], testDouble[2], testDouble[6], testDouble[4], testDouble[7],
+ testDouble[4], testDouble[2], testDouble[8], testDouble[7], testDouble[2] };
+ arrayEqualityCheck(expected4, new FieldsFromTuple(13, 4, 5, 4, 2, 8, 6, 2, 8, 3, 5, 2, 16,
+ 4, 3, 2, 6, 4, 7, 4, 2, 8, 7, 2).extract(currentTuple));
+ }
+
+ private void arrayEqualityCheck(double[] array1, double[] array2) {
+ assertEquals("The result arrays must have the same length", array1.length, array2.length);
+ for (int i = 0; i < array1.length; i++) {
+ assertEquals("Unequal fields at position " + i, array1[i], array2[i], 0d);
+ }
+ }
+
+ private static final Class<?>[] CLASSES = new Class<?>[] { Tuple1.class, Tuple2.class,
+ Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
+ Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
+ Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
+ Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
+ Tuple24.class, Tuple25.class };
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java
new file mode 100644
index 0000000..4773f4e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class CountEvictionPolicyTest {
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Test
+ public void testCountEvictionPolicy() {
+ List<Integer> tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+ int counter;
+
+ // The count policy should not care about the triggered parameter
+ // Therefore its value switches after each use in this test.
+ boolean triggered = false;
+ // the size of the buffer should not matter as well!
+
+ // Test count of different sizes (0..9)
+ for (int i = 0; i < 10; i++) {
+ EvictionPolicy evictionPolicy = new CountEvictionPolicy(i, i);
+ counter = 0;
+
+ // Test first i steps (should not evict)
+ for (int j = 0; j < i; j++) {
+ counter++;
+ assertEquals("Evictionpolicy with count of " + i + " evicted tuples at add nr. "
+ + counter + ". It should not evict for the first " + i + " adds.", 0,
+ evictionPolicy.notifyEviction(tuples.get(j), (triggered = !triggered),
+ tuples.get(Math.abs((i - j)) % 10)));
+ }
+
+ // Test the next three evictions
+ for (int j = 0; j < 3; j++) {
+ // The first add should evict now
+ counter++;
+ assertEquals("Evictionpolicy with count of " + i
+ + " did not evict correct number of tuples at the expected pos " + counter
+ + ".", i, evictionPolicy.notifyEviction(tuples.get(j),
+ (triggered = !triggered), tuples.get(Math.abs((i - j)) % 10)));
+
+ // the next i-1 adds should not evict
+ for (int k = 0; k < i - 1; k++) {
+ counter++;
+ assertEquals("Evictionpolicy with count of " + i
+ + " evicted tuples at add nr. " + counter, 0,
+ evictionPolicy.notifyEviction(tuples.get(j), (triggered = !triggered),
+ tuples.get(Math.abs((i - j)) % 10)));
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testCountEvictionPolicyStartValuesAndEvictionAmount() {
+
+ // The count policy should not care about the triggered parameter
+ // Therefore its value switches after each use in this test.
+ boolean triggered = false;
+ // the size of the buffer should not matter as well!
+
+ List<Integer> tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ // Text different eviction amounts (0..3)
+ for (int x = 0; x < 4; x++) {
+
+ // Test count of different sizes (0..9)
+ for (int i = 0; i < 10; i++) {
+
+ int counter = 0;
+
+ // Test different start values (-5..5)
+ for (int j = -5; i < 6; i++) {
+ EvictionPolicy evictionPolicy = new CountEvictionPolicy(i, x, j);
+ // Add tuples without eviction
+ for (int k = 0; k < ((i - j > 0) ? i - j : 0); k++) {
+ counter++;
+ assertEquals("Evictionpolicy with count of " + i
+ + " did not evict correct number of tuples at the expected pos "
+ + counter + ".", 0, evictionPolicy.notifyEviction(
+ tuples.get(Math.abs(j)), (triggered = !triggered),
+ tuples.get(Math.abs((i - j)) % 10)));
+ }
+ // Expect eviction
+ counter++;
+ assertEquals("Evictionpolicy with count of " + i
+ + " did not evict correct number of tuples at the expected pos "
+ + counter + ".", x, evictionPolicy.notifyEviction(
+ tuples.get(Math.abs(j)), (triggered = !triggered),
+ tuples.get(Math.abs((i - j)) % 10)));
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java
new file mode 100644
index 0000000..b7120e7
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.junit.Test;
+
+public class CountTriggerPolicyTest {
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testCountTriggerPolicy() {
+
+ List tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+ int counter;
+
+ // Test count of different sizes (0..9)
+ for (int i = 0; i < 10; i++) {
+ TriggerPolicy triggerPolicy = Count.of(i).toTrigger();
+ counter=0;
+
+ // Test first i steps (should not trigger)
+ for (int j = 0; j < i; j++) {
+ counter++;
+ assertFalse("Triggerpolicy with count of " + i + " triggered at add nr. " + counter
+ + ". It should not trigger for the first " + i + " adds.",
+ triggerPolicy.notifyTrigger(tuples.get(j)));
+ }
+
+ // Test the next three triggers
+ for (int j = 0; j < 3; j++) {
+ // The first add should trigger now
+ counter++;
+ assertTrue("Triggerpolicy with count of " + i
+ + " did not trigger at the expected pos " + counter + ".",
+ triggerPolicy.notifyTrigger(tuples.get(j)));
+
+ // the next i-1 adds should not trigger
+ for (int k = 0; k < i - 1; k++) {
+ counter++;
+ assertFalse("Triggerpolicy with count of " + i + " triggered at add nr. "
+ + counter, triggerPolicy.notifyTrigger(tuples.get(k)));
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testCountTriggerPolicyStartValues() {
+
+ List tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ // Test count of different sizes (0..9)
+ for (int i = 0; i < 10; i++) {
+
+ // Test different start values (-5..5)
+ for (int j = -5; i < 6; i++) {
+ TriggerPolicy triggerPolicy = new CountTriggerPolicy(i, j);
+ // Add tuples without trigger
+ for (int k = 0; k < ((i - j > 0) ? i - j : 0); k++) {
+ assertFalse("Triggerpolicy with count of " + i + " and start value of " + j
+ + " triggered at add nr. " + (k + 1),
+ triggerPolicy.notifyTrigger(tuples.get(k % 10)));
+ }
+ // Expect trigger
+ assertTrue("Triggerpolicy with count of " + i + "and start value of " + j
+ + " did not trigger at the expected position.",
+ triggerPolicy.notifyTrigger(tuples.get(0)));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java
new file mode 100644
index 0000000..424eb57
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
+import org.apache.flink.streaming.api.windowing.policy.DeltaPolicy;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Arrays;
+
+import static org.junit.Assert.*;
+
+public class DeltaPolicyTest {
+
+ @SuppressWarnings({ "serial", "unchecked", "rawtypes" })
+ @Test
+ public void testDelta() {
+ DeltaPolicy deltaPolicy = new DeltaPolicy(new DeltaFunction<Tuple2<Integer, Integer>>() {
+ @Override
+ public double getDelta(Tuple2<Integer, Integer> oldDataPoint, Tuple2<Integer, Integer> newDataPoint) {
+ return (double) newDataPoint.f0 - oldDataPoint.f0;
+ }
+ }, new Tuple2(0, 0), 2);
+
+ List<Tuple2> tuples = Arrays.asList(
+ new Tuple2(1, 0),
+ new Tuple2(2, 0),
+ new Tuple2(3, 0),
+ new Tuple2(6, 0));
+
+ assertFalse(deltaPolicy.notifyTrigger(tuples.get(0)));
+ assertEquals(0, deltaPolicy.notifyEviction(tuples.get(0), false, 0));
+
+ assertFalse(deltaPolicy.notifyTrigger(tuples.get(1)));
+ assertEquals(0, deltaPolicy.notifyEviction(tuples.get(1), false, 1));
+
+ assertTrue(deltaPolicy.notifyTrigger(tuples.get(2)));
+ assertEquals(1, deltaPolicy.notifyEviction(tuples.get(2), true, 2));
+
+ assertTrue(deltaPolicy.notifyTrigger(tuples.get(3)));
+ assertEquals(2, deltaPolicy.notifyEviction(tuples.get(3), true, 2));
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java
new file mode 100644
index 0000000..8224b12
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.windowing.extractor.FieldFromTuple;
+import org.junit.Test;
+
+public class PunctuationPolicyTest {
+
+ // This value should not effect the policy. It is changed at each call to
+ // verify this.
+ private boolean triggered = false;
+
+ @Test
+ public void PunctuationTriggerTestWithoutExtraction() {
+ PunctuationPolicy<Object, Object> policy = new PunctuationPolicy<Object, Object>(
+ new TestObject(0));
+ assertTrue("The present punctuation was not detected. (POS 1)",
+ policy.notifyTrigger(new TestObject(0)));
+ assertFalse("There was a punctuation detected which wasn't present. (POS 2)",
+ policy.notifyTrigger(new TestObject(1)));
+ }
+
+ @Test
+ public void PunctuationTriggerTestWithExtraction() {
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ PunctuationPolicy<Tuple2<Object, Object>, Object> policy = new PunctuationPolicy<Tuple2<Object, Object>, Object>(
+ new TestObject(0), new FieldFromTuple(0));
+ assertTrue("The present punctuation was not detected. (POS 3)",
+ policy.notifyTrigger(new Tuple2<Object, Object>(new TestObject(0),
+ new TestObject(1))));
+ assertFalse("There was a punctuation detected which wasn't present. (POS 4)",
+ policy.notifyTrigger(new Tuple2<Object, Object>(new TestObject(1),
+ new TestObject(0))));
+ }
+
+ @Test
+ public void PunctuationEvictionTestWithoutExtraction() {
+ // The current buffer size should not effect the test. It's therefore
+ // always 0 here.
+
+ PunctuationPolicy<Object, Object> policy = new PunctuationPolicy<Object, Object>(
+ new TestObject(0));
+ assertEquals(
+ "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 5)",
+ 0, policy.notifyEviction(new TestObject(0), (triggered = !triggered), 0));
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < i; j++) {
+ assertEquals("There was a punctuation detected which wasn't present. (POS 6)", 0,
+ policy.notifyEviction(new TestObject(1), (triggered = !triggered), 0));
+ }
+ assertEquals(
+ "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 7)",
+ i + 1, policy.notifyEviction(new TestObject(0), (triggered = !triggered), 0));
+ }
+ }
+
+ @Test
+ public void PunctuationEvictionTestWithExtraction() {
+ // The current buffer size should not effect the test. It's therefore
+ // always 0 here.
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ PunctuationPolicy<Tuple2<Object, Object>, Object> policy = new PunctuationPolicy<Tuple2<Object, Object>, Object>(
+ new TestObject(0), new FieldFromTuple(0));
+ assertEquals(
+ "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 10)",
+ 0, policy.notifyEviction(new Tuple2<Object, Object>(new TestObject(0),
+ new TestObject(1)), (triggered = !triggered), 0));
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < i; j++) {
+ assertEquals("There was a punctuation detected which wasn't present. (POS 9)", 0,
+ policy.notifyEviction(new Tuple2<Object, Object>(new TestObject(1),
+ new TestObject(0)), (triggered = !triggered), 0));
+ }
+ assertEquals(
+ "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 10)",
+ i + 1, policy.notifyEviction(new Tuple2<Object, Object>(new TestObject(0),
+ new TestObject(1)), (triggered = !triggered), 0));
+ }
+ }
+
+ private class TestObject {
+
+ private int id;
+
+ public TestObject(int id) {
+ this.id = id;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof TestObject && ((TestObject) o).getId() == this.id) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public int getId() {
+ return id;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
new file mode 100644
index 0000000..b5d502b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.util.LinkedList;
+
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TimeEvictionPolicyTest {
+
+ @Test
+ public void timeEvictionTest() {
+ // create some test data
+ Integer[] times = { 1, 3, 4, 6, 7, 9, 14, 20, 21, 22, 30, 31, 33, 36, 40, 41, 42, 43, 44,
+ 45, 47, 55 };
+ Integer[] numToDelete = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 3 };
+
+ // create a timestamp
+ @SuppressWarnings("serial")
+ Timestamp<Integer> timeStamp = new Timestamp<Integer>() {
+
+ @Override
+ public long getTimestamp(Integer value) {
+ return value;
+ }
+
+ };
+
+ // test different granularity
+ for (long granularity = 0; granularity < 40; granularity++) {
+ // create policy
+ TimeEvictionPolicy<Integer> policy = new TimeEvictionPolicy<Integer>(granularity,
+ new TimestampWrapper<Integer>(timeStamp, 0));
+
+ // The trigger status should not effect the policy. Therefore, it's
+ // value is changed after each usage.
+ boolean triggered = false;
+
+ // The eviction should work similar with both, fake and real
+ // elements. Which kind is used is changed on every 3rd element in
+ // this test.
+ int fakeAndRealCounter = 0;
+ boolean fake = false;
+
+ // test by adding values
+ LinkedList<Integer> buffer = new LinkedList<Integer>();
+ for (int i = 0; i < times.length; i++) {
+
+ // check if the current element should be a fake
+ fakeAndRealCounter++;
+ if (fakeAndRealCounter > 2) {
+ fake = !fake;
+ fakeAndRealCounter = 0;
+ }
+
+ int result;
+
+ if (fake) {
+ // Notify eviction with fake element
+ result = policy.notifyEvictionWithFakeElement(times[i], buffer.size());
+ } else {
+ // Notify eviction with real element
+ result = policy.notifyEviction(times[i], (triggered = !triggered),
+ buffer.size());
+ }
+
+ // handle correctness of eviction
+ for (; result > 0 && !buffer.isEmpty(); result--) {
+ if (buffer.getFirst() <= times[i] - granularity) {
+ buffer.removeFirst();
+ } else {
+ fail("The policy wanted to evict time " + buffer.getFirst()
+ + " while the current time was " + times[i]
+ + "and the granularity was " + granularity);
+ }
+ }
+
+ // test that all required evictions have been done
+ if (!buffer.isEmpty()) {
+ assertTrue("The policy did not evict " + buffer.getFirst()
+ + " while the current time was " + times[i]
+ + " and the granularity was " + granularity,
+ (buffer.getFirst() >= times[i] - granularity));
+ }
+
+ // test influence of other evictions
+ for (int j = numToDelete[i % numToDelete.length]; j > 0; j--) {
+ if (!buffer.isEmpty()) {
+ buffer.removeFirst();
+ }
+ }
+
+ // add current element to buffer if it is no fake
+ if (!fake) {
+ buffer.add(times[i]);
+ }
+
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
new file mode 100644
index 0000000..2bdbd96
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.junit.Test;
+
+public class TimeTriggerPolicyTest {
+
+ @Test
+ public void timeTriggerRegularNotifyTest() {
+ // create some test data
+ Integer[] times = { 1, 3, 4, 6, 7, 9, 14, 20, 21, 22, 30 };
+
+ // create a timestamp
+ @SuppressWarnings("serial")
+ Timestamp<Integer> timeStamp = new Timestamp<Integer>() {
+
+ @Override
+ public long getTimestamp(Integer value) {
+ return value;
+ }
+
+ };
+
+ // test different granularity
+ for (long granularity = 0; granularity < 31; granularity++) {
+ // create policy
+
+ TriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(granularity,
+ new TimestampWrapper<Integer>(timeStamp, 0));
+
+ // remember window border
+ long currentTime = 0;
+
+ // test by adding values
+ for (int i = 0; i < times.length; i++) {
+ boolean result = policy.notifyTrigger(times[i]);
+ // start time is included, but end time is excluded: >=
+ if (times[i] >= currentTime + granularity) {
+ if (granularity != 0) {
+ currentTime = times[i] - ((times[i] - currentTime) % granularity);
+ }
+ assertTrue("The policy did not trigger at pos " + i + " (current time border: "
+ + currentTime + "; current granularity: " + granularity
+ + "; data point time: " + times[i] + ")", result);
+ } else {
+ assertFalse("The policy triggered wrong at pos " + i
+ + " (current time border: " + currentTime + "; current granularity: "
+ + granularity + "; data point time: " + times[i] + ")", result);
+ }
+ }
+ }
+
+ }
+
+ @Test
+ public void timeTriggerPreNotifyTest() {
+ // create some test data
+ Integer[] times = { 1, 3, 20, 26 };
+
+ // create a timestamp
+ @SuppressWarnings("serial")
+ Timestamp<Integer> timeStamp = new Timestamp<Integer>() {
+
+ @Override
+ public long getTimestamp(Integer value) {
+ return value;
+ }
+
+ };
+
+ // create policy
+ TimeTriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(5,
+ new TimestampWrapper<Integer>(timeStamp, 0));
+
+ // expected result
+ Long[][] result = { {}, {}, { 4L, 9L, 14L, 19L }, { 24L } };
+
+ // call policy
+ for (int i = 0; i < times.length; i++) {
+ arrayEqualityCheck(result[i], policy.preNotifyTrigger(times[i]));
+ policy.notifyTrigger(times[i]);
+ }
+ }
+
+ private void arrayEqualityCheck(Object[] array1, Object[] array2) {
+ assertEquals("The result arrays must have the same length", array1.length, array2.length);
+ for (int i = 0; i < array1.length; i++) {
+ assertEquals("Unequal fields at position " + i, array1[i], array2[i]);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java
new file mode 100644
index 0000000..8cb03a7
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class TumblingEvictionPolicyTest {
+
+ @Test
+ public void testTumblingEviction() {
+ EvictionPolicy<Integer> policy = new TumblingEvictionPolicy<Integer>();
+
+ int counter = 0;
+
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < i; j++) {
+ assertEquals(0, policy.notifyEviction(0, false, counter++));
+ }
+ assertEquals(counter, policy.notifyEviction(0, true, counter));
+ counter = 1;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/BroadcastPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/BroadcastPartitionerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/BroadcastPartitionerTest.java
new file mode 100644
index 0000000..de6c200
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/BroadcastPartitionerTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.junit.Before;
+import org.junit.Test;
+
+public class BroadcastPartitionerTest {
+
+ private BroadcastPartitioner<Tuple> broadcastPartitioner1;
+ private BroadcastPartitioner<Tuple> broadcastPartitioner2;
+ private BroadcastPartitioner<Tuple> broadcastPartitioner3;
+
+ private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
+ private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(null);
+
+ @Before
+ public void setPartitioner() {
+ broadcastPartitioner1 = new BroadcastPartitioner<Tuple>();
+ broadcastPartitioner2 = new BroadcastPartitioner<Tuple>();
+ broadcastPartitioner3 = new BroadcastPartitioner<Tuple>();
+
+ }
+
+ @Test
+ public void testSelectChannels() {
+ int[] first = new int[] { 0 };
+ int[] second = new int[] { 0, 1 };
+ int[] sixth = new int[] { 0, 1, 2, 3, 4, 5 };
+ sd.setInstance(streamRecord);
+ assertArrayEquals(first, broadcastPartitioner1.selectChannels(sd, 1));
+ assertArrayEquals(second, broadcastPartitioner2.selectChannels(sd, 2));
+ assertArrayEquals(sixth, broadcastPartitioner3.selectChannels(sd, 6));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java
new file mode 100644
index 0000000..0675242
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DistributePartitionerTest {
+
+ private DistributePartitioner<Tuple> distributePartitioner;
+ private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
+ private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
+ null);
+
+ @Before
+ public void setPartitioner() {
+ distributePartitioner = new DistributePartitioner<Tuple>(false);
+ }
+
+ @Test
+ public void testSelectChannelsLength() {
+ sd.setInstance(streamRecord);
+ assertEquals(1, distributePartitioner.selectChannels(sd, 1).length);
+ assertEquals(1, distributePartitioner.selectChannels(sd, 2).length);
+ assertEquals(1, distributePartitioner.selectChannels(sd, 1024).length);
+ }
+
+ @Test
+ public void testSelectChannelsInterval() {
+ sd.setInstance(streamRecord);
+ assertEquals(0, distributePartitioner.selectChannels(sd, 3)[0]);
+ assertEquals(1, distributePartitioner.selectChannels(sd, 3)[0]);
+ assertEquals(2, distributePartitioner.selectChannels(sd, 3)[0]);
+ assertEquals(0, distributePartitioner.selectChannels(sd, 3)[0]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java
new file mode 100644
index 0000000..b56649b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FieldsPartitionerTest {
+
+ private FieldsPartitioner<Tuple> fieldsPartitioner;
+ private StreamRecord<Tuple> streamRecord1 = new StreamRecord<Tuple>()
+ .setObject(new Tuple2<String, Integer>("test", 0));
+ private StreamRecord<Tuple> streamRecord2 = new StreamRecord<Tuple>()
+ .setObject(new Tuple2<String, Integer>("test", 42));
+ private SerializationDelegate<StreamRecord<Tuple>> sd1 = new SerializationDelegate<StreamRecord<Tuple>>(
+ null);
+ private SerializationDelegate<StreamRecord<Tuple>> sd2 = new SerializationDelegate<StreamRecord<Tuple>>(
+ null);
+
+ @Before
+ public void setPartitioner() {
+ fieldsPartitioner = new FieldsPartitioner<Tuple>(new KeySelector<Tuple, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple value) throws Exception {
+ return value.getField(0);
+ }
+ });
+ }
+
+ @Test
+ public void testSelectChannelsLength() {
+ sd1.setInstance(streamRecord1);
+ assertEquals(1, fieldsPartitioner.selectChannels(sd1, 1).length);
+ assertEquals(1, fieldsPartitioner.selectChannels(sd1, 2).length);
+ assertEquals(1, fieldsPartitioner.selectChannels(sd1, 1024).length);
+ }
+
+ @Test
+ public void testSelectChannelsGrouping() {
+ sd1.setInstance(streamRecord1);
+ sd2.setInstance(streamRecord2);
+
+ assertArrayEquals(fieldsPartitioner.selectChannels(sd1, 1),
+ fieldsPartitioner.selectChannels(sd2, 1));
+ assertArrayEquals(fieldsPartitioner.selectChannels(sd1, 2),
+ fieldsPartitioner.selectChannels(sd2, 2));
+ assertArrayEquals(fieldsPartitioner.selectChannels(sd1, 1024),
+ fieldsPartitioner.selectChannels(sd2, 1024));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java
new file mode 100644
index 0000000..b381d85
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ForwardPartitionerTest {
+
+ private DistributePartitioner<Tuple> forwardPartitioner;
+ private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
+ private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
+ null);
+
+ @Before
+ public void setPartitioner() {
+ forwardPartitioner = new DistributePartitioner<Tuple>(true);
+ }
+
+ @Test
+ public void testSelectChannelsLength() {
+ sd.setInstance(streamRecord);
+ assertEquals(1, forwardPartitioner.selectChannels(sd, 1).length);
+ assertEquals(1, forwardPartitioner.selectChannels(sd, 2).length);
+ assertEquals(1, forwardPartitioner.selectChannels(sd, 1024).length);
+ }
+
+ @Test
+ public void testSelectChannelsInterval() {
+ sd.setInstance(streamRecord);
+ assertEquals(0, forwardPartitioner.selectChannels(sd, 1)[0]);
+ assertEquals(1, forwardPartitioner.selectChannels(sd, 2)[0]);
+ assertEquals(2, forwardPartitioner.selectChannels(sd, 1024)[0]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/GlobalPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/GlobalPartitionerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/GlobalPartitionerTest.java
new file mode 100644
index 0000000..eebda64
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/GlobalPartitionerTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GlobalPartitionerTest {
+
+ private GlobalPartitioner<Tuple> globalPartitioner;
+ private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
+ private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
+ null);
+
+ @Before
+ public void setPartitioner() {
+ globalPartitioner = new GlobalPartitioner<Tuple>();
+ }
+
+ @Test
+ public void testSelectChannels() {
+ int[] result = new int[] { 0 };
+
+ sd.setInstance(streamRecord);
+
+ assertArrayEquals(result, globalPartitioner.selectChannels(sd, 1));
+ assertArrayEquals(result, globalPartitioner.selectChannels(sd, 2));
+ assertArrayEquals(result, globalPartitioner.selectChannels(sd, 1024));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ShufflePartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ShufflePartitionerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ShufflePartitionerTest.java
new file mode 100644
index 0000000..3c03d07
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ShufflePartitionerTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.partitioner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ShufflePartitionerTest {
+
+ private ShufflePartitioner<Tuple> shufflePartitioner;
+ private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
+ private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
+ null);
+
+ @Before
+ public void setPartitioner() {
+ shufflePartitioner = new ShufflePartitioner<Tuple>();
+ }
+
+ @Test
+ public void testSelectChannelsLength() {
+ sd.setInstance(streamRecord);
+ assertEquals(1, shufflePartitioner.selectChannels(sd, 1).length);
+ assertEquals(1, shufflePartitioner.selectChannels(sd, 2).length);
+ assertEquals(1, shufflePartitioner.selectChannels(sd, 1024).length);
+ }
+
+ @Test
+ public void testSelectChannelsInterval() {
+ sd.setInstance(streamRecord);
+ assertEquals(0, shufflePartitioner.selectChannels(sd, 1)[0]);
+
+ assertTrue(0 <= shufflePartitioner.selectChannels(sd, 2)[0]);
+ assertTrue(2 > shufflePartitioner.selectChannels(sd, 2)[0]);
+
+ assertTrue(0 <= shufflePartitioner.selectChannels(sd, 1024)[0]);
+ assertTrue(1024 > shufflePartitioner.selectChannels(sd, 1024)[0]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/MapStateTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/MapStateTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/MapStateTest.java
new file mode 100644
index 0000000..194403c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/MapStateTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.flink.streaming.state.checkpoint.MapCheckpoint;
+import org.apache.flink.streaming.state.checkpoint.StateCheckpoint;
+import org.junit.Test;
+
+public class MapStateTest {
+
+ @Test
+ public void testMapState() {
+
+ Map<String, Integer> map = new HashMap<String, Integer>();
+ map.put("a", 1);
+ map.put("b", 2);
+ map.put("c", 3);
+ map.remove("a");
+
+ MapState<String, Integer> mapState = new MapState<String, Integer>();
+ mapState.put("a", 1);
+ mapState.put("b", 2);
+ mapState.put("c", 3);
+
+ assertEquals(1, (int) mapState.remove("a"));
+ assertEquals(null, mapState.remove("a"));
+ assertEquals(2, mapState.size());
+ assertEquals(map, mapState.state);
+ assertEquals(map.entrySet(), mapState.entrySet());
+ assertTrue(mapState.containsKey("b"));
+ assertFalse(mapState.containsKey("a"));
+
+ assertEquals(2, mapState.updatedItems.size());
+ assertEquals(1, mapState.removedItems.size());
+
+ Map<String, Integer> map2 = new HashMap<String, Integer>();
+ map2.put("a", 0);
+ map2.put("e", -1);
+
+ mapState.putAll(map2);
+
+ assertEquals(4, mapState.updatedItems.size());
+ assertEquals(0, mapState.removedItems.size());
+
+ mapState.clear();
+ assertEquals(new HashMap<String, Integer>(), mapState.state);
+ assertTrue(mapState.clear);
+ assertEquals(0, mapState.updatedItems.size());
+ assertEquals(0, mapState.removedItems.size());
+
+ mapState.putAll(map);
+ assertEquals(map.keySet(), mapState.updatedItems);
+
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMapStateCheckpointing() {
+
+ Map<String, Integer> map = new HashMap<String, Integer>();
+ map.put("a", 1);
+ map.put("b", 2);
+ map.put("c", 3);
+
+ MapState<String, Integer> mapState = new MapState<String, Integer>();
+ mapState.putAll(map);
+
+ StateCheckpoint<Map<String, Integer>> mcp = mapState.checkpoint();
+ assertEquals(map, mcp.getCheckpointedState());
+
+ Map<String, Integer> map2 = new HashMap<String, Integer>();
+ map2.put("a", 0);
+ map2.put("e", -1);
+
+ mapState.put("a", 0);
+ mapState.put("e", -1);
+ mapState.remove("b");
+ StateCheckpoint<Map<String, Integer>> mcp2 = new MapCheckpoint<String, Integer>(mapState);
+ assertEquals(map2, mcp2.getCheckpointedState());
+ mcp.update(mcp2);
+ assertEquals(mapState.state, mcp.getCheckpointedState());
+
+ mapState.clear();
+ mapState.put("a", 1);
+ mapState.put("a", 2);
+ mapState.put("b", -3);
+ mapState.put("c", 0);
+ mapState.remove("b");
+ mcp.update(mapState.checkpoint());
+ assertEquals(mapState.state, mcp.getCheckpointedState());
+
+ MapState<String, Integer> mapState2 = (MapState<String, Integer>) new MapState<String, Integer>()
+ .restore(mcp);
+
+ assertTrue(mapState2.stateEquals(mapState));
+ mapState2.reBuild(mapState2.repartition(10));
+ assertTrue(mapState2.stateEquals(mapState));
+
+ MapState<Integer, Integer> mapState3 = new MapState<Integer, Integer>();
+ mapState3.put(1, 1);
+ mapState3.put(2, 1);
+
+ mapState3.reBuild(mapState3.repartition(2)[0]);
+ assertTrue(mapState3.containsKey(2));
+ assertFalse(mapState3.containsKey(1));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/OperatorStateTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/OperatorStateTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/OperatorStateTest.java
new file mode 100644
index 0000000..6cb8f51
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/state/OperatorStateTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.state;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.flink.streaming.state.checkpoint.StateCheckpoint;
+import org.junit.Test;
+
+public class OperatorStateTest {
+
+ @Test
+ public void testOperatorState() {
+ OperatorState<Integer> os = new SimpleState<Integer>(5);
+
+ StateCheckpoint<Integer> scp = os.checkpoint();
+
+ assertTrue(os.stateEquals(new SimpleState<Integer>().restore(scp)));
+
+ assertEquals((Integer) 5, os.getState());
+
+ os.setState(10);
+
+ assertEquals((Integer) 10, os.getState());
+
+ scp.update(os.checkpoint());
+
+ assertEquals((Integer)10,scp.getCheckpointedState());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
new file mode 100644
index 0000000..ea94f98
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.api.streamvertex.StreamTaskContext;
+import org.apache.flink.streaming.io.CoReaderIterator;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.MutableObjectIterator;
+
+public class MockCoContext<IN1, IN2, OUT> implements StreamTaskContext<OUT> {
+ // private Collection<IN1> input1;
+ // private Collection<IN2> input2;
+ private Iterator<IN1> inputIterator1;
+ private Iterator<IN2> inputIterator2;
+ private List<OUT> outputs;
+
+ private Collector<OUT> collector;
+ private StreamRecordSerializer<IN1> inDeserializer1;
+ private CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>> mockIterator;
+ private StreamRecordSerializer<IN2> inDeserializer2;
+
+ public MockCoContext(Collection<IN1> input1, Collection<IN2> input2) {
+
+ if (input1.isEmpty() || input2.isEmpty()) {
+ throw new RuntimeException("Inputs must not be empty");
+ }
+
+ this.inputIterator1 = input1.iterator();
+ this.inputIterator2 = input2.iterator();
+
+ TypeInformation<IN1> inTypeInfo1 = TypeExtractor.getForObject(input1.iterator().next());
+ inDeserializer1 = new StreamRecordSerializer<IN1>(inTypeInfo1);
+ TypeInformation<IN2> inTypeInfo2 = TypeExtractor.getForObject(input2.iterator().next());
+ inDeserializer2 = new StreamRecordSerializer<IN2>(inTypeInfo2);
+
+ mockIterator = new MockCoReaderIterator(inDeserializer1, inDeserializer2);
+
+ outputs = new ArrayList<OUT>();
+ collector = new MockCollector<OUT>(outputs);
+ }
+
+ private int currentInput = 1;
+ private StreamRecord<IN1> reuse1;
+ private StreamRecord<IN2> reuse2;
+
+ private class MockCoReaderIterator extends
+ CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>> {
+
+ public MockCoReaderIterator(TypeSerializer<StreamRecord<IN1>> serializer1,
+ TypeSerializer<StreamRecord<IN2>> serializer2) {
+ super(null, serializer1, serializer2);
+ reuse1 = inDeserializer1.createInstance();
+ reuse2 = inDeserializer2.createInstance();
+ }
+
+ @Override
+ public int next(StreamRecord<IN1> target1, StreamRecord<IN2> target2) throws IOException {
+ this.delegate1.setInstance(target1);
+ this.delegate2.setInstance(target2);
+
+ int inputNumber = nextRecord();
+ target1.setObject(reuse1.getObject());
+ target2.setObject(reuse2.getObject());
+
+ return inputNumber;
+ }
+ }
+
+ private Integer nextRecord() {
+ if (inputIterator1.hasNext() && inputIterator2.hasNext()) {
+ switch (currentInput) {
+ case 1:
+ return next1();
+ case 2:
+ return next2();
+ default:
+ return 0;
+ }
+ }
+
+ if (inputIterator1.hasNext()) {
+ return next1();
+ }
+
+ if (inputIterator2.hasNext()) {
+ return next2();
+ }
+
+ return 0;
+ }
+
+ private int next1() {
+ reuse1 = inDeserializer1.createInstance();
+ reuse1.setObject(inputIterator1.next());
+ currentInput = 2;
+ return 1;
+ }
+
+ private int next2() {
+ reuse2 = inDeserializer2.createInstance();
+ reuse2.setObject(inputIterator2.next());
+ currentInput = 1;
+ return 2;
+ }
+
+ public List<OUT> getOutputs() {
+ return outputs;
+ }
+
+ public Collector<OUT> getCollector() {
+ return collector;
+ }
+
+ public StreamRecordSerializer<IN1> getInDeserializer1() {
+ return inDeserializer1;
+ }
+
+ public StreamRecordSerializer<IN2> getInDeserializer2() {
+ return inDeserializer2;
+ }
+
+ public CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>> getIterator() {
+ return mockIterator;
+ }
+
+ public static <IN1, IN2, OUT> List<OUT> createAndExecute(CoInvokable<IN1, IN2, OUT> invokable,
+ List<IN1> input1, List<IN2> input2) {
+ MockCoContext<IN1, IN2, OUT> mockContext = new MockCoContext<IN1, IN2, OUT>(input1, input2);
+ invokable.setup(mockContext);
+
+ try {
+ invokable.open(null);
+ invokable.invoke();
+ invokable.close();
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot invoke invokable.", e);
+ }
+
+ return mockContext.getOutputs();
+ }
+
+ @Override
+ public StreamConfig getConfig() {
+ return null;
+ }
+
+ @Override
+ public ClassLoader getUserCodeClassLoader() {
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <X> MutableObjectIterator<X> getInput(int index) {
+ switch (index) {
+ case 0:
+ return (MutableObjectIterator<X>) inputIterator1;
+ case 1:
+ return (MutableObjectIterator<X>) inputIterator2;
+ default:
+ throw new IllegalArgumentException("CoStreamVertex has only 2 inputs");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <X> StreamRecordSerializer<X> getInputSerializer(int index) {
+ switch (index) {
+ case 0:
+ return (StreamRecordSerializer<X>) inDeserializer1;
+ case 1:
+ return (StreamRecordSerializer<X>) inDeserializer2;
+ default:
+ throw new IllegalArgumentException("CoStreamVertex has only 2 inputs");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <X, Y> CoReaderIterator<X, Y> getCoReader() {
+ return (CoReaderIterator<X, Y>) mockIterator;
+ }
+
+ @Override
+ public Collector<OUT> getOutputCollector() {
+ return collector;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCollector.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCollector.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCollector.java
new file mode 100644
index 0000000..e8b96c5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCollector.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import java.io.Serializable;
+import java.util.Collection;
+
+import org.apache.commons.lang3.SerializationUtils;
+import org.apache.flink.util.Collector;
+
+public class MockCollector<T> implements Collector<T> {
+ private Collection<T> outputs;
+
+ public MockCollector(Collection<T> outputs) {
+ this.outputs = outputs;
+ }
+
+ @Override
+ public void collect(T record) {
+ T copied = SerializationUtils.deserialize(SerializationUtils
+ .serialize((Serializable) record));
+ outputs.add(copied);
+ }
+
+ @Override
+ public void close() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
new file mode 100644
index 0000000..5537052
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.api.streamvertex.StreamTaskContext;
+import org.apache.flink.streaming.io.CoReaderIterator;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.MutableObjectIterator;
+
+public class MockContext<IN, OUT> implements StreamTaskContext<OUT> {
+ private Collection<IN> inputs;
+ private List<OUT> outputs;
+
+ private Collector<OUT> collector;
+ private StreamRecordSerializer<IN> inDeserializer;
+ private MutableObjectIterator<StreamRecord<IN>> iterator;
+
+ public MockContext(Collection<IN> inputs) {
+ this.inputs = inputs;
+ if (inputs.isEmpty()) {
+ throw new RuntimeException("Inputs must not be empty");
+ }
+
+ TypeInformation<IN> inTypeInfo = TypeExtractor.getForObject(inputs.iterator().next());
+ inDeserializer = new StreamRecordSerializer<IN>(inTypeInfo);
+
+ iterator = new MockInputIterator();
+ outputs = new ArrayList<OUT>();
+ collector = new MockCollector<OUT>(outputs);
+ }
+
+ private class MockInputIterator implements MutableObjectIterator<StreamRecord<IN>> {
+ Iterator<IN> listIterator;
+
+ public MockInputIterator() {
+ listIterator = inputs.iterator();
+ }
+
+ @Override
+ public StreamRecord<IN> next(StreamRecord<IN> reuse) throws IOException {
+ if (listIterator.hasNext()) {
+ reuse.setObject(listIterator.next());
+ } else {
+ reuse = null;
+ }
+ return reuse;
+ }
+
+ @Override
+ public StreamRecord<IN> next() throws IOException {
+ if (listIterator.hasNext()) {
+ StreamRecord<IN> result = new StreamRecord<IN>();
+ result.setObject(listIterator.next());
+ return result;
+ } else {
+ return null;
+ }
+ }
+ }
+
+ public List<OUT> getOutputs() {
+ return outputs;
+ }
+
+ public Collector<OUT> getCollector() {
+ return collector;
+ }
+
+ public StreamRecordSerializer<IN> getInDeserializer() {
+ return inDeserializer;
+ }
+
+ public MutableObjectIterator<StreamRecord<IN>> getIterator() {
+ return iterator;
+ }
+
+ public static <IN, OUT> List<OUT> createAndExecute(StreamInvokable<IN, OUT> invokable,
+ List<IN> inputs) {
+ MockContext<IN, OUT> mockContext = new MockContext<IN, OUT>(inputs);
+ invokable.setup(mockContext);
+ try {
+ invokable.open(null);
+ invokable.invoke();
+ invokable.close();
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot invoke invokable.", e);
+ }
+
+ return mockContext.getOutputs();
+ }
+
+ @Override
+ public StreamConfig getConfig() {
+ return null;
+ }
+
+ @Override
+ public ClassLoader getUserCodeClassLoader() {
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <X> MutableObjectIterator<X> getInput(int index) {
+ if (index == 0) {
+ return (MutableObjectIterator<X>) iterator;
+ } else {
+ throw new IllegalArgumentException("There is only 1 input");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <X> StreamRecordSerializer<X> getInputSerializer(int index) {
+ if (index == 0) {
+ return (StreamRecordSerializer<X>) inDeserializer;
+ } else {
+ throw new IllegalArgumentException("There is only 1 input");
+ }
+ }
+
+ @Override
+ public Collector<OUT> getOutputCollector() {
+ return collector;
+ }
+
+ @Override
+ public <X, Y> CoReaderIterator<X, Y> getCoReader() {
+ throw new IllegalArgumentException("CoReader not available");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockRecordWriterFactory.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockRecordWriterFactory.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockRecordWriterFactory.java
new file mode 100644
index 0000000..88673a3
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockRecordWriterFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.mock;
+
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamvertex.MockRecordWriter;
+import org.mockito.Mockito;
+
+public class MockRecordWriterFactory {
+
+ @SuppressWarnings("unchecked")
+ public static MockRecordWriter create() {
+ MockRecordWriter recWriter = mock(MockRecordWriter.class);
+
+ Mockito.when(recWriter.initList()).thenCallRealMethod();
+ doCallRealMethod().when(recWriter).emit(Mockito.any(SerializationDelegate.class));
+
+ recWriter.initList();
+
+ return recWriter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockSource.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockSource.java
new file mode 100644
index 0000000..bb92e8e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockSource.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+
+public class MockSource<T> {
+
+ public static <T> List<T> createAndExecute(SourceFunction<T> source) {
+ List<T> outputs = new ArrayList<T>();
+ try {
+ source.invoke(new MockCollector<T>(outputs));
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot invoke source.", e);
+ }
+ return outputs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
new file mode 100644
index 0000000..95a5d9b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import akka.actor.ActorRef;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.client.JobClient;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.util.ForkableFlinkMiniCluster;
+
+public class TestStreamEnvironment extends StreamExecutionEnvironment {
+ private static final String DEFAULT_JOBNAME = "TestStreamingJob";
+ private static final String CANNOT_EXECUTE_EMPTY_JOB = "Cannot execute empty job";
+
+ private long memorySize;
+
+ public TestStreamEnvironment(int degreeOfParallelism, long memorySize){
+ this.setDegreeOfParallelism(degreeOfParallelism);
+
+ this.memorySize = memorySize;
+ }
+
+ @Override
+ public void execute() throws Exception {
+ execute(DEFAULT_JOBNAME);
+ }
+
+ @Override
+ public void execute(String jobName) throws Exception {
+ JobGraph jobGraph = streamGraph.getJobGraph(jobName);
+
+ Configuration configuration = jobGraph.getJobConfiguration();
+
+ configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS,
+ getDegreeOfParallelism());
+ configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, memorySize);
+
+ ForkableFlinkMiniCluster cluster = new ForkableFlinkMiniCluster(configuration);
+
+ try{
+ ActorRef client = cluster.getJobClient();
+ JobClient.submitJobAndWait(jobGraph, false, client, cluster.timeout());
+ }catch(JobExecutionException e){
+ if(e.getMessage().contains("GraphConversionException")){
+ throw new Exception(CANNOT_EXECUTE_EMPTY_JOB, e);
+ }else{
+ throw e;
+ }
+ }finally{
+ cluster.stop();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties b/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..2fb9345
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties
@@ -0,0 +1,19 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/logback-test.xml b/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..4f56748
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/logback-test.xml
@@ -0,0 +1,30 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+ <logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>
+</configuration>
\ No newline at end of file
[51/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
[FLINK-1452] Rename 'flink-addons' to 'flink-staging'
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f94112fb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f94112fb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f94112fb
Branch: refs/heads/master
Commit: f94112fbbaaf2ecc6a9ecb314a5565203ce779a7
Parents: 563e546
Author: Robert Metzger <rm...@apache.org>
Authored: Fri Jan 30 15:16:46 2015 +0100
Committer: Robert Metzger <rm...@apache.org>
Committed: Mon Feb 2 19:40:25 2015 +0100
----------------------------------------------------------------------
docs/internal_add_operator.md | 2 +-
docs/internal_general_arch.md | 3 +-
docs/streaming_guide.md | 6 +-
flink-addons/flink-avro/pom.xml | 182 ---
.../apache/flink/api/avro/DataInputDecoder.java | 213 ---
.../flink/api/avro/DataOutputEncoder.java | 183 ---
.../api/avro/FSDataInputStreamWrapper.java | 68 -
.../api/io/avro/example/AvroTypeExample.java | 111 --
.../apache/flink/api/io/avro/example/User.java | 269 ----
.../flink/api/java/io/AvroInputFormat.java | 124 --
.../flink/api/java/io/AvroOutputFormat.java | 100 --
.../src/test/assembly/test-assembly.xml | 36 -
.../api/avro/AvroExternalJarProgramITCase.java | 71 -
.../flink/api/avro/AvroOutputFormatTest.java | 173 --
.../flink/api/avro/EncoderDecoderTest.java | 519 ------
.../avro/testjar/AvroExternalJarProgram.java | 219 ---
.../api/io/avro/AvroRecordInputFormatTest.java | 167 --
.../flink/api/io/avro/generated/Colors.java | 32 -
.../flink/api/io/avro/generated/User.java | 755 ---------
.../io/AvroInputFormatTypeExtractionTest.java | 79 -
.../src/test/resources/avro/user.avsc | 19 -
.../src/test/resources/log4j-test.properties | 27 -
.../src/test/resources/logback-test.xml | 29 -
.../flink-avro/src/test/resources/testdata.avro | Bin 4572 -> 0 bytes
flink-addons/flink-hadoop-compatibility/pom.xml | 80 -
.../mapred/HadoopInputFormat.java | 297 ----
.../mapred/HadoopMapFunction.java | 131 --
.../mapred/HadoopOutputFormat.java | 183 ---
.../mapred/HadoopReduceCombineFunction.java | 162 --
.../mapred/HadoopReduceFunction.java | 138 --
.../example/HadoopMapredCompatWordCount.java | 133 --
.../mapred/record/HadoopDataSink.java | 109 --
.../mapred/record/HadoopDataSource.java | 82 -
.../mapred/record/HadoopRecordInputFormat.java | 174 ---
.../mapred/record/HadoopRecordOutputFormat.java | 156 --
.../datatypes/DefaultFlinkTypeConverter.java | 95 --
.../datatypes/DefaultHadoopTypeConverter.java | 83 -
.../record/datatypes/FlinkTypeConverter.java | 43 -
.../datatypes/HadoopFileOutputCommitter.java | 196 ---
.../record/datatypes/HadoopTypeConverter.java | 42 -
.../datatypes/WritableComparableWrapper.java | 40 -
.../record/datatypes/WritableWrapper.java | 71 -
.../datatypes/WritableWrapperConverter.java | 45 -
.../mapred/record/example/WordCount.java | 184 ---
.../example/WordCountWithOutputFormat.java | 173 --
.../mapred/utils/HadoopUtils.java | 87 --
.../mapred/wrapper/HadoopDummyProgressable.java | 33 -
.../mapred/wrapper/HadoopDummyReporter.java | 70 -
.../mapred/wrapper/HadoopInputSplit.java | 138 --
.../mapred/wrapper/HadoopOutputCollector.java | 64 -
.../wrapper/HadoopTupleUnwrappingIterator.java | 94 --
.../mapreduce/HadoopInputFormat.java | 338 ----
.../mapreduce/HadoopOutputFormat.java | 226 ---
.../mapreduce/example/WordCount.java | 120 --
.../mapreduce/utils/HadoopUtils.java | 83 -
.../mapreduce/wrapper/HadoopInputSplit.java | 125 --
.../mapred/HadoopIOFormatsITCase.java | 221 ---
.../mapred/HadoopInputFormatTest.java | 82 -
.../mapred/HadoopMapFunctionITCase.java | 182 ---
.../mapred/HadoopMapredITCase.java | 47 -
.../HadoopReduceCombineFunctionITCase.java | 265 ----
.../mapred/HadoopReduceFunctionITCase.java | 213 ---
.../mapred/HadoopTestData.java | 62 -
.../record/HadoopRecordInputOutputITCase.java | 54 -
.../HadoopTupleUnwrappingIteratorTest.java | 137 --
.../mapreduce/HadoopInputFormatTest.java | 84 -
.../mapreduce/HadoopInputOutputITCase.java | 47 -
.../src/test/resources/log4j-test.properties | 27 -
.../src/test/resources/logback-test.xml | 29 -
flink-addons/flink-hbase/pom.xml | 165 --
.../flink/addons/hbase/TableInputFormat.java | 255 ---
.../flink/addons/hbase/TableInputSplit.java | 168 --
.../addons/hbase/example/HBaseReadExample.java | 93 --
.../src/test/resources/hbase-site.xml | 43 -
.../src/test/resources/log4j.properties | 23 -
flink-addons/flink-jdbc/pom.xml | 64 -
.../flink/api/java/io/jdbc/JDBCInputFormat.java | 358 -----
.../api/java/io/jdbc/JDBCOutputFormat.java | 270 ----
.../api/java/io/jdbc/example/JDBCExample.java | 101 --
.../java/record/io/jdbc/JDBCInputFormat.java | 389 -----
.../java/record/io/jdbc/JDBCOutputFormat.java | 359 -----
.../record/io/jdbc/example/JDBCExample.java | 136 --
.../api/java/io/jdbc/JDBCInputFormatTest.java | 195 ---
.../api/java/io/jdbc/JDBCOutputFormatTest.java | 240 ---
.../java/record/io/jdbc/DevNullLogStream.java | 30 -
.../record/io/jdbc/JDBCInputFormatTest.java | 214 ---
.../record/io/jdbc/JDBCOutputFormatTest.java | 225 ---
.../src/test/resources/log4j-test.properties | 19 -
.../src/test/resources/logback-test.xml | 29 -
flink-addons/flink-spargel/pom.xml | 60 -
.../flink/spargel/java/MessageIterator.java | 58 -
.../flink/spargel/java/MessagingFunction.java | 284 ----
.../apache/flink/spargel/java/OutgoingEdge.java | 64 -
.../spargel/java/VertexCentricIteration.java | 599 -------
.../spargel/java/VertexUpdateFunction.java | 145 --
.../examples/SpargelConnectedComponents.java | 79 -
.../spargel/java/examples/SpargelPageRank.java | 117 --
.../SpargelPageRankCountingVertices.java | 154 --
.../apache/flink/spargel/java/record/Edge.java | 43 -
.../spargel/java/record/MessageIterator.java | 59 -
.../spargel/java/record/MessagingFunction.java | 163 --
.../spargel/java/record/SpargelIteration.java | 289 ----
.../java/record/VertexUpdateFunction.java | 90 --
.../flink/spargel/java/SpargelCompilerTest.java | 183 ---
.../spargel/java/SpargelTranslationTest.java | 211 ---
.../SpargelConnectedComponentsITCase.java | 81 -
.../src/test/resources/log4j-test.properties | 19 -
.../src/test/resources/logback-test.xml | 29 -
.../flink-streaming-connectors/pom.xml | 243 ---
.../streaming/connectors/ConnectorSource.java | 42 -
.../db/CustomSerializationDBState.java | 31 -
.../streaming/connectors/db/DBSerializer.java | 36 -
.../flink/streaming/connectors/db/DBState.java | 29 -
.../connectors/db/DBStateIterator.java | 31 -
.../connectors/db/DBStateWithIterator.java | 25 -
.../connectors/db/DefaultDBSerializer.java | 35 -
.../streaming/connectors/db/LevelDBState.java | 125 --
.../streaming/connectors/db/MemcachedState.java | 73 -
.../streaming/connectors/db/RedisState.java | 109 --
.../streaming/connectors/flume/FlumeSink.java | 141 --
.../streaming/connectors/flume/FlumeSource.java | 141 --
.../connectors/flume/FlumeTopology.java | 49 -
.../connectors/json/JSONParseFlatMap.java | 144 --
.../streaming/connectors/json/JSONParser.java | 175 ---
.../streaming/connectors/kafka/KafkaSink.java | 83 -
.../streaming/connectors/kafka/KafkaSource.java | 96 --
.../connectors/kafka/KafkaTopology.java | 57 -
.../streaming/connectors/rabbitmq/RMQSink.java | 111 --
.../connectors/rabbitmq/RMQSource.java | 121 --
.../connectors/rabbitmq/RMQTopology.java | 52 -
.../connectors/twitter/TwitterSource.java | 283 ----
.../connectors/twitter/TwitterStreaming.java | 99 --
.../connectors/twitter/TwitterTopology.java | 92 --
.../connectors/util/DeserializationSchema.java | 42 -
.../streaming/connectors/util/RawSchema.java | 39 -
.../connectors/util/SerializationSchema.java | 33 -
.../connectors/util/SimpleStringSchema.java | 40 -
.../streaming/connectors/db/DBStateTest.java | 105 --
.../connectors/json/JSONParserTest.java | 74 -
.../connectors/json/JSONParserTest2.java | 95 --
.../src/test/resources/log4j-test.properties | 19 -
.../src/test/resources/logback-test.xml | 30 -
.../flink-streaming-core/pom.xml | 75 -
.../flink/streaming/api/StreamConfig.java | 406 -----
.../apache/flink/streaming/api/StreamGraph.java | 620 --------
.../api/StreamingJobGraphGenerator.java | 293 ----
.../api/collector/CollectorWrapper.java | 49 -
.../api/collector/DirectedCollectorWrapper.java | 131 --
.../streaming/api/collector/OutputSelector.java | 44 -
.../streaming/api/collector/StreamOutput.java | 86 -
.../api/datastream/ConnectedDataStream.java | 419 -----
.../streaming/api/datastream/DataStream.java | 1271 ---------------
.../api/datastream/DataStreamSink.java | 46 -
.../api/datastream/DataStreamSource.java | 48 -
.../api/datastream/GroupedDataStream.java | 205 ---
.../api/datastream/IterativeDataStream.java | 107 --
.../datastream/SingleOutputStreamOperator.java | 174 ---
.../api/datastream/SplitDataStream.java | 66 -
.../api/datastream/StreamProjection.java | 1471 ------------------
.../api/datastream/WindowedDataStream.java | 658 --------
.../temporaloperator/StreamCrossOperator.java | 101 --
.../temporaloperator/StreamJoinOperator.java | 272 ----
.../temporaloperator/TemporalOperator.java | 124 --
.../temporaloperator/TemporalWindow.java | 45 -
.../api/environment/LocalStreamEnvironment.java | 47 -
.../environment/RemoteStreamEnvironment.java | 127 --
.../environment/StreamContextEnvironment.java | 76 -
.../environment/StreamExecutionEnvironment.java | 618 --------
.../api/environment/StreamPlanEnvironment.java | 64 -
.../aggregation/AggregationFunction.java | 35 -
.../aggregation/ComparableAggregator.java | 236 ---
.../api/function/aggregation/Comparator.java | 104 --
.../api/function/aggregation/SumAggregator.java | 171 --
.../api/function/aggregation/SumFunction.java | 102 --
.../api/function/co/CoFlatMapFunction.java | 42 -
.../api/function/co/CoMapFunction.java | 41 -
.../api/function/co/CoReduceFunction.java | 107 --
.../api/function/co/CoWindowFunction.java | 30 -
.../api/function/co/CrossWindowFunction.java | 44 -
.../api/function/co/JoinWindowFunction.java | 55 -
.../api/function/co/RichCoFlatMapFunction.java | 40 -
.../api/function/co/RichCoMapFunction.java | 40 -
.../api/function/co/RichCoReduceFunction.java | 40 -
.../api/function/co/RichCoWindowFunction.java | 34 -
.../api/function/sink/FileSinkFunction.java | 118 --
.../function/sink/FileSinkFunctionByMillis.java | 59 -
.../api/function/sink/PrintSinkFunction.java | 98 --
.../api/function/sink/RichSinkFunction.java | 28 -
.../api/function/sink/SinkFunction.java | 28 -
.../api/function/sink/WriteFormat.java | 43 -
.../api/function/sink/WriteFormatAsCsv.java | 49 -
.../api/function/sink/WriteFormatAsText.java | 47 -
.../api/function/sink/WriteSinkFunction.java | 92 --
.../sink/WriteSinkFunctionByMillis.java | 50 -
.../function/source/FileMonitoringFunction.java | 123 --
.../api/function/source/FileReadFunction.java | 51 -
.../api/function/source/FileSourceFunction.java | 142 --
.../function/source/FromElementsFunction.java | 49 -
.../function/source/GenSequenceFunction.java | 53 -
.../function/source/GenericSourceFunction.java | 25 -
.../function/source/ParallelSourceFunction.java | 26 -
.../source/RichParallelSourceFunction.java | 27 -
.../api/function/source/RichSourceFunction.java | 27 -
.../source/SocketTextStreamFunction.java | 86 -
.../api/function/source/SourceFunction.java | 29 -
.../api/invokable/ChainableInvokable.java | 39 -
.../streaming/api/invokable/SinkInvokable.java | 50 -
.../api/invokable/SourceInvokable.java | 44 -
.../api/invokable/StreamInvokable.java | 186 ---
.../api/invokable/operator/BatchIterator.java | 25 -
.../invokable/operator/CounterInvokable.java | 43 -
.../api/invokable/operator/FilterInvokable.java | 55 -
.../invokable/operator/FlatMapInvokable.java | 51 -
.../operator/GroupedReduceInvokable.java | 59 -
.../operator/GroupedWindowInvokable.java | 506 ------
.../api/invokable/operator/MapInvokable.java | 50 -
.../invokable/operator/ProjectInvokable.java | 61 -
.../operator/StreamReduceInvokable.java | 69 -
.../operator/WindowGroupReduceInvokable.java | 51 -
.../api/invokable/operator/WindowInvokable.java | 382 -----
.../operator/WindowReduceInvokable.java | 59 -
.../operator/co/CoFlatMapInvokable.java | 54 -
.../operator/co/CoGroupedReduceInvokable.java | 88 --
.../api/invokable/operator/co/CoInvokable.java | 133 --
.../invokable/operator/co/CoMapInvokable.java | 54 -
.../operator/co/CoReduceInvokable.java | 70 -
.../operator/co/CoWindowInvokable.java | 197 ---
.../api/streamrecord/StreamRecord.java | 134 --
.../streamrecord/StreamRecordSerializer.java | 114 --
.../flink/streaming/api/streamrecord/UID.java | 122 --
.../api/streamvertex/CoStreamVertex.java | 142 --
.../api/streamvertex/InputHandler.java | 96 --
.../api/streamvertex/OutputHandler.java | 242 ---
.../api/streamvertex/StreamIterationHead.java | 101 --
.../api/streamvertex/StreamIterationTail.java | 103 --
.../api/streamvertex/StreamTaskContext.java | 40 -
.../api/streamvertex/StreamVertex.java | 156 --
.../api/streamvertex/StreamVertexException.java | 68 -
.../streamvertex/StreamingRuntimeContext.java | 88 --
.../windowing/deltafunction/CosineDistance.java | 87 --
.../windowing/deltafunction/DeltaFunction.java | 44 -
.../deltafunction/EuclideanDistance.java | 58 -
.../ExtractionAwareDeltaFunction.java | 90 --
.../api/windowing/extractor/ArrayFromTuple.java | 74 -
.../extractor/ConcatinatedExtract.java | 68 -
.../api/windowing/extractor/Extractor.java | 43 -
.../api/windowing/extractor/FieldFromArray.java | 59 -
.../api/windowing/extractor/FieldFromTuple.java | 58 -
.../windowing/extractor/FieldsFromArray.java | 67 -
.../windowing/extractor/FieldsFromTuple.java | 53 -
.../streaming/api/windowing/helper/Count.java | 101 --
.../streaming/api/windowing/helper/Delta.java | 91 --
.../api/windowing/helper/SystemTimestamp.java | 37 -
.../streaming/api/windowing/helper/Time.java | 166 --
.../api/windowing/helper/Timestamp.java | 39 -
.../api/windowing/helper/TimestampWrapper.java | 44 -
.../api/windowing/helper/WindowingHelper.java | 39 -
.../ActiveCloneableEvictionPolicyWrapper.java | 62 -
.../windowing/policy/ActiveEvictionPolicy.java | 52 -
.../policy/ActiveEvictionPolicyWrapper.java | 64 -
.../windowing/policy/ActiveTriggerCallback.java | 45 -
.../windowing/policy/ActiveTriggerPolicy.java | 81 -
.../policy/CloneableEvictionPolicy.java | 45 -
.../policy/CloneableTriggerPolicy.java | 45 -
.../windowing/policy/CountEvictionPolicy.java | 119 --
.../windowing/policy/CountTriggerPolicy.java | 87 --
.../api/windowing/policy/DeltaPolicy.java | 112 --
.../api/windowing/policy/EvictionPolicy.java | 47 -
.../api/windowing/policy/PunctuationPolicy.java | 116 --
.../windowing/policy/TimeEvictionPolicy.java | 137 --
.../api/windowing/policy/TimeTriggerPolicy.java | 196 ---
.../api/windowing/policy/TriggerPolicy.java | 54 -
.../policy/TumblingEvictionPolicy.java | 90 --
.../flink/streaming/io/BlockingQueueBroker.java | 41 -
.../flink/streaming/io/CoReaderIterator.java | 57 -
.../flink/streaming/io/CoRecordReader.java | 221 ---
.../flink/streaming/io/StreamRecordWriter.java | 83 -
.../partitioner/BroadcastPartitioner.java | 55 -
.../partitioner/DistributePartitioner.java | 46 -
.../partitioner/FieldsPartitioner.java | 50 -
.../partitioner/GlobalPartitioner.java | 38 -
.../partitioner/ShufflePartitioner.java | 49 -
.../partitioner/StreamPartitioner.java | 44 -
.../flink/streaming/state/CircularFifoList.java | 112 --
.../flink/streaming/state/GraphState.java | 61 -
.../apache/flink/streaming/state/MapState.java | 201 ---
.../streaming/state/NullableCircularBuffer.java | 362 -----
.../flink/streaming/state/OperatorState.java | 105 --
.../streaming/state/PartitionableState.java | 60 -
.../flink/streaming/state/SimpleState.java | 53 -
.../state/checkpoint/MapCheckpoint.java | 72 -
.../state/checkpoint/StateCheckpoint.java | 82 -
.../flink/streaming/util/ClusterUtil.java | 76 -
.../streaming/util/keys/KeySelectorUtil.java | 126 --
.../streaming/api/AggregationFunctionTest.java | 318 ----
.../apache/flink/streaming/api/IterateTest.java | 95 --
.../flink/streaming/api/OutputSplitterTest.java | 185 ---
.../apache/flink/streaming/api/PrintTest.java | 57 -
.../apache/flink/streaming/api/SourceTest.java | 70 -
.../streaming/api/WindowCrossJoinTest.java | 154 --
.../api/collector/DirectedOutputTest.java | 112 --
.../api/collector/OutputSelectorTest.java | 58 -
.../api/collector/StreamCollectorTest.java | 52 -
.../api/invokable/operator/CoFlatMapTest.java | 84 -
.../invokable/operator/CoGroupedReduceTest.java | 125 --
.../api/invokable/operator/CoMapTest.java | 57 -
.../invokable/operator/CoStreamReduceTest.java | 71 -
.../api/invokable/operator/CoWindowTest.java | 182 ---
.../operator/CounterInvokableTest.java | 39 -
.../api/invokable/operator/FilterTest.java | 51 -
.../api/invokable/operator/FlatMapTest.java | 54 -
.../operator/GroupedReduceInvokableTest.java | 62 -
.../operator/GroupedWindowInvokableTest.java | 574 -------
.../api/invokable/operator/MapTest.java | 49 -
.../api/invokable/operator/ProjectTest.java | 67 -
.../invokable/operator/StreamReduceTest.java | 54 -
.../invokable/operator/WindowInvokableTest.java | 261 ----
.../streaming/api/streamrecord/UIDTest.java | 53 -
.../api/streamvertex/MockRecordWriter.java | 45 -
.../api/streamvertex/StreamVertexTest.java | 177 ---
.../deltafunction/CosineDistanceTest.java | 70 -
.../deltafunction/EuclideanDistanceTest.java | 71 -
.../windowing/extractor/ArrayFromTupleTest.java | 118 --
.../extractor/ConcatinatedExtractTest.java | 83 -
.../windowing/extractor/FieldFromArrayTest.java | 55 -
.../windowing/extractor/FieldFromTupleTest.java | 84 -
.../extractor/FieldsFromArrayTest.java | 108 --
.../extractor/FieldsFromTupleTest.java | 106 --
.../policy/CountEvictionPolicyTest.java | 119 --
.../policy/CountTriggerPolicyTest.java | 94 --
.../api/windowing/policy/DeltaPolicyTest.java | 62 -
.../windowing/policy/PunctuationPolicyTest.java | 123 --
.../policy/TimeEvictionPolicyTest.java | 121 --
.../windowing/policy/TimeTriggerPolicyTest.java | 114 --
.../policy/TumblingEvictionPolicyTest.java | 41 -
.../partitioner/BroadcastPartitionerTest.java | 55 -
.../partitioner/DistributePartitionerTest.java | 56 -
.../partitioner/FieldsPartitionerTest.java | 76 -
.../partitioner/ForwardPartitionerTest.java | 55 -
.../partitioner/GlobalPartitionerTest.java | 50 -
.../partitioner/ShufflePartitionerTest.java | 60 -
.../flink/streaming/state/MapStateTest.java | 130 --
.../streaming/state/OperatorStateTest.java | 48 -
.../flink/streaming/util/MockCoContext.java | 217 ---
.../flink/streaming/util/MockCollector.java | 43 -
.../flink/streaming/util/MockContext.java | 159 --
.../streaming/util/MockRecordWriterFactory.java | 40 -
.../apache/flink/streaming/util/MockSource.java | 36 -
.../streaming/util/TestStreamEnvironment.java | 72 -
.../src/test/resources/log4j-test.properties | 19 -
.../src/test/resources/logback-test.xml | 30 -
.../flink-streaming-examples/pom.xml | 518 ------
.../examples/iteration/IterateExample.java | 168 --
.../streaming/examples/join/WindowJoin.java | 190 ---
.../ml/IncrementalLearningSkeleton.java | 213 ---
.../socket/SocketTextStreamWordCount.java | 108 --
.../examples/twitter/TwitterStream.java | 202 ---
.../twitter/util/TwitterStreamData.java | 255 ---
.../examples/windowing/DeltaExtractExample.java | 125 --
.../windowing/MultiplePoliciesExample.java | 139 --
.../examples/windowing/SlidingExample.java | 139 --
.../windowing/TimeWindowingExample.java | 152 --
.../windowing/TopSpeedWindowingExample.java | 128 --
.../examples/wordcount/PojoExample.java | 185 ---
.../streaming/examples/wordcount/WordCount.java | 148 --
.../socket/SocketTextStreamWordCount.scala | 91 --
.../examples/windowing/TopSpeedWindowing.scala | 94 --
.../scala/examples/windowing/WindowJoin.scala | 72 -
.../flink-streaming-scala/pom.xml | 217 ---
.../scala/ScalaStreamingAggregator.java | 111 --
.../api/scala/ConnectedDataStream.scala | 380 -----
.../flink/streaming/api/scala/DataStream.scala | 573 -------
.../streaming/api/scala/SplitDataStream.scala | 39 -
.../api/scala/StreamCrossOperator.scala | 120 --
.../api/scala/StreamExecutionEnvironment.scala | 289 ----
.../api/scala/StreamJoinOperator.scala | 212 ---
.../streaming/api/scala/TemporalOperator.scala | 43 -
.../api/scala/WindowedDataStream.scala | 248 ---
.../flink/streaming/api/scala/package.scala | 70 -
.../streaming/api/scala/windowing/Delta.scala | 47 -
.../streaming/api/scala/windowing/Time.scala | 54 -
flink-addons/flink-streaming/pom.xml | 75 -
flink-addons/flink-tachyon/pom.xml | 117 --
.../java/org/apache/flink/tachyon/HDFSTest.java | 160 --
.../tachyon/TachyonFileSystemWrapperTest.java | 167 --
.../src/test/resources/log4j.properties | 23 -
.../src/test/resources/tachyonHadoopConf.xml | 28 -
flink-addons/pom.xml | 64 -
flink-staging/flink-avro/pom.xml | 182 +++
.../apache/flink/api/avro/DataInputDecoder.java | 213 +++
.../flink/api/avro/DataOutputEncoder.java | 183 +++
.../api/avro/FSDataInputStreamWrapper.java | 68 +
.../api/io/avro/example/AvroTypeExample.java | 111 ++
.../apache/flink/api/io/avro/example/User.java | 269 ++++
.../flink/api/java/io/AvroInputFormat.java | 124 ++
.../flink/api/java/io/AvroOutputFormat.java | 100 ++
.../src/test/assembly/test-assembly.xml | 36 +
.../api/avro/AvroExternalJarProgramITCase.java | 71 +
.../flink/api/avro/AvroOutputFormatTest.java | 173 ++
.../flink/api/avro/EncoderDecoderTest.java | 519 ++++++
.../avro/testjar/AvroExternalJarProgram.java | 219 +++
.../api/io/avro/AvroRecordInputFormatTest.java | 167 ++
.../flink/api/io/avro/generated/Colors.java | 32 +
.../flink/api/io/avro/generated/User.java | 755 +++++++++
.../io/AvroInputFormatTypeExtractionTest.java | 79 +
.../src/test/resources/avro/user.avsc | 19 +
.../src/test/resources/log4j-test.properties | 27 +
.../src/test/resources/logback-test.xml | 29 +
.../flink-avro/src/test/resources/testdata.avro | Bin 0 -> 4572 bytes
.../flink-hadoop-compatibility/pom.xml | 80 +
.../mapred/HadoopInputFormat.java | 297 ++++
.../mapred/HadoopMapFunction.java | 131 ++
.../mapred/HadoopOutputFormat.java | 183 +++
.../mapred/HadoopReduceCombineFunction.java | 162 ++
.../mapred/HadoopReduceFunction.java | 138 ++
.../example/HadoopMapredCompatWordCount.java | 133 ++
.../mapred/record/HadoopDataSink.java | 109 ++
.../mapred/record/HadoopDataSource.java | 82 +
.../mapred/record/HadoopRecordInputFormat.java | 174 +++
.../mapred/record/HadoopRecordOutputFormat.java | 156 ++
.../datatypes/DefaultFlinkTypeConverter.java | 95 ++
.../datatypes/DefaultHadoopTypeConverter.java | 83 +
.../record/datatypes/FlinkTypeConverter.java | 43 +
.../datatypes/HadoopFileOutputCommitter.java | 196 +++
.../record/datatypes/HadoopTypeConverter.java | 42 +
.../datatypes/WritableComparableWrapper.java | 40 +
.../record/datatypes/WritableWrapper.java | 71 +
.../datatypes/WritableWrapperConverter.java | 45 +
.../mapred/record/example/WordCount.java | 184 +++
.../example/WordCountWithOutputFormat.java | 173 ++
.../mapred/utils/HadoopUtils.java | 87 ++
.../mapred/wrapper/HadoopDummyProgressable.java | 33 +
.../mapred/wrapper/HadoopDummyReporter.java | 70 +
.../mapred/wrapper/HadoopInputSplit.java | 138 ++
.../mapred/wrapper/HadoopOutputCollector.java | 64 +
.../wrapper/HadoopTupleUnwrappingIterator.java | 94 ++
.../mapreduce/HadoopInputFormat.java | 338 ++++
.../mapreduce/HadoopOutputFormat.java | 226 +++
.../mapreduce/example/WordCount.java | 120 ++
.../mapreduce/utils/HadoopUtils.java | 83 +
.../mapreduce/wrapper/HadoopInputSplit.java | 125 ++
.../mapred/HadoopIOFormatsITCase.java | 221 +++
.../mapred/HadoopInputFormatTest.java | 82 +
.../mapred/HadoopMapFunctionITCase.java | 182 +++
.../mapred/HadoopMapredITCase.java | 47 +
.../HadoopReduceCombineFunctionITCase.java | 265 ++++
.../mapred/HadoopReduceFunctionITCase.java | 213 +++
.../mapred/HadoopTestData.java | 62 +
.../record/HadoopRecordInputOutputITCase.java | 54 +
.../HadoopTupleUnwrappingIteratorTest.java | 137 ++
.../mapreduce/HadoopInputFormatTest.java | 84 +
.../mapreduce/HadoopInputOutputITCase.java | 47 +
.../src/test/resources/log4j-test.properties | 27 +
.../src/test/resources/logback-test.xml | 29 +
flink-staging/flink-hbase/pom.xml | 165 ++
.../flink/addons/hbase/TableInputFormat.java | 255 +++
.../flink/addons/hbase/TableInputSplit.java | 168 ++
.../addons/hbase/example/HBaseReadExample.java | 93 ++
.../src/test/resources/hbase-site.xml | 43 +
.../src/test/resources/log4j.properties | 23 +
flink-staging/flink-jdbc/pom.xml | 64 +
.../flink/api/java/io/jdbc/JDBCInputFormat.java | 358 +++++
.../api/java/io/jdbc/JDBCOutputFormat.java | 270 ++++
.../api/java/io/jdbc/example/JDBCExample.java | 101 ++
.../java/record/io/jdbc/JDBCInputFormat.java | 389 +++++
.../java/record/io/jdbc/JDBCOutputFormat.java | 359 +++++
.../record/io/jdbc/example/JDBCExample.java | 136 ++
.../api/java/io/jdbc/JDBCInputFormatTest.java | 195 +++
.../api/java/io/jdbc/JDBCOutputFormatTest.java | 240 +++
.../java/record/io/jdbc/DevNullLogStream.java | 30 +
.../record/io/jdbc/JDBCInputFormatTest.java | 214 +++
.../record/io/jdbc/JDBCOutputFormatTest.java | 225 +++
.../src/test/resources/log4j-test.properties | 19 +
.../src/test/resources/logback-test.xml | 29 +
flink-staging/flink-spargel/pom.xml | 60 +
.../flink/spargel/java/MessageIterator.java | 58 +
.../flink/spargel/java/MessagingFunction.java | 284 ++++
.../apache/flink/spargel/java/OutgoingEdge.java | 64 +
.../spargel/java/VertexCentricIteration.java | 599 +++++++
.../spargel/java/VertexUpdateFunction.java | 145 ++
.../examples/SpargelConnectedComponents.java | 79 +
.../spargel/java/examples/SpargelPageRank.java | 117 ++
.../SpargelPageRankCountingVertices.java | 154 ++
.../apache/flink/spargel/java/record/Edge.java | 43 +
.../spargel/java/record/MessageIterator.java | 59 +
.../spargel/java/record/MessagingFunction.java | 163 ++
.../spargel/java/record/SpargelIteration.java | 289 ++++
.../java/record/VertexUpdateFunction.java | 90 ++
.../flink/spargel/java/SpargelCompilerTest.java | 183 +++
.../spargel/java/SpargelTranslationTest.java | 211 +++
.../SpargelConnectedComponentsITCase.java | 81 +
.../src/test/resources/log4j-test.properties | 19 +
.../src/test/resources/logback-test.xml | 29 +
.../flink-streaming-connectors/pom.xml | 243 +++
.../streaming/connectors/ConnectorSource.java | 42 +
.../db/CustomSerializationDBState.java | 31 +
.../streaming/connectors/db/DBSerializer.java | 36 +
.../flink/streaming/connectors/db/DBState.java | 29 +
.../connectors/db/DBStateIterator.java | 31 +
.../connectors/db/DBStateWithIterator.java | 25 +
.../connectors/db/DefaultDBSerializer.java | 35 +
.../streaming/connectors/db/LevelDBState.java | 125 ++
.../streaming/connectors/db/MemcachedState.java | 73 +
.../streaming/connectors/db/RedisState.java | 109 ++
.../streaming/connectors/flume/FlumeSink.java | 141 ++
.../streaming/connectors/flume/FlumeSource.java | 141 ++
.../connectors/flume/FlumeTopology.java | 49 +
.../connectors/json/JSONParseFlatMap.java | 144 ++
.../streaming/connectors/json/JSONParser.java | 175 +++
.../streaming/connectors/kafka/KafkaSink.java | 83 +
.../streaming/connectors/kafka/KafkaSource.java | 96 ++
.../connectors/kafka/KafkaTopology.java | 57 +
.../streaming/connectors/rabbitmq/RMQSink.java | 111 ++
.../connectors/rabbitmq/RMQSource.java | 121 ++
.../connectors/rabbitmq/RMQTopology.java | 52 +
.../connectors/twitter/TwitterSource.java | 283 ++++
.../connectors/twitter/TwitterStreaming.java | 99 ++
.../connectors/twitter/TwitterTopology.java | 92 ++
.../connectors/util/DeserializationSchema.java | 42 +
.../streaming/connectors/util/RawSchema.java | 39 +
.../connectors/util/SerializationSchema.java | 33 +
.../connectors/util/SimpleStringSchema.java | 40 +
.../streaming/connectors/db/DBStateTest.java | 105 ++
.../connectors/json/JSONParserTest.java | 74 +
.../connectors/json/JSONParserTest2.java | 95 ++
.../src/test/resources/log4j-test.properties | 19 +
.../src/test/resources/logback-test.xml | 30 +
.../flink-streaming-core/pom.xml | 75 +
.../flink/streaming/api/StreamConfig.java | 406 +++++
.../apache/flink/streaming/api/StreamGraph.java | 620 ++++++++
.../api/StreamingJobGraphGenerator.java | 293 ++++
.../api/collector/CollectorWrapper.java | 49 +
.../api/collector/DirectedCollectorWrapper.java | 131 ++
.../streaming/api/collector/OutputSelector.java | 44 +
.../streaming/api/collector/StreamOutput.java | 86 +
.../api/datastream/ConnectedDataStream.java | 419 +++++
.../streaming/api/datastream/DataStream.java | 1271 +++++++++++++++
.../api/datastream/DataStreamSink.java | 46 +
.../api/datastream/DataStreamSource.java | 48 +
.../api/datastream/GroupedDataStream.java | 205 +++
.../api/datastream/IterativeDataStream.java | 107 ++
.../datastream/SingleOutputStreamOperator.java | 174 +++
.../api/datastream/SplitDataStream.java | 66 +
.../api/datastream/StreamProjection.java | 1471 ++++++++++++++++++
.../api/datastream/WindowedDataStream.java | 658 ++++++++
.../temporaloperator/StreamCrossOperator.java | 101 ++
.../temporaloperator/StreamJoinOperator.java | 272 ++++
.../temporaloperator/TemporalOperator.java | 124 ++
.../temporaloperator/TemporalWindow.java | 45 +
.../api/environment/LocalStreamEnvironment.java | 47 +
.../environment/RemoteStreamEnvironment.java | 127 ++
.../environment/StreamContextEnvironment.java | 76 +
.../environment/StreamExecutionEnvironment.java | 618 ++++++++
.../api/environment/StreamPlanEnvironment.java | 64 +
.../aggregation/AggregationFunction.java | 35 +
.../aggregation/ComparableAggregator.java | 236 +++
.../api/function/aggregation/Comparator.java | 104 ++
.../api/function/aggregation/SumAggregator.java | 171 ++
.../api/function/aggregation/SumFunction.java | 102 ++
.../api/function/co/CoFlatMapFunction.java | 42 +
.../api/function/co/CoMapFunction.java | 41 +
.../api/function/co/CoReduceFunction.java | 107 ++
.../api/function/co/CoWindowFunction.java | 30 +
.../api/function/co/CrossWindowFunction.java | 44 +
.../api/function/co/JoinWindowFunction.java | 55 +
.../api/function/co/RichCoFlatMapFunction.java | 40 +
.../api/function/co/RichCoMapFunction.java | 40 +
.../api/function/co/RichCoReduceFunction.java | 40 +
.../api/function/co/RichCoWindowFunction.java | 34 +
.../api/function/sink/FileSinkFunction.java | 118 ++
.../function/sink/FileSinkFunctionByMillis.java | 59 +
.../api/function/sink/PrintSinkFunction.java | 98 ++
.../api/function/sink/RichSinkFunction.java | 28 +
.../api/function/sink/SinkFunction.java | 28 +
.../api/function/sink/WriteFormat.java | 43 +
.../api/function/sink/WriteFormatAsCsv.java | 49 +
.../api/function/sink/WriteFormatAsText.java | 47 +
.../api/function/sink/WriteSinkFunction.java | 92 ++
.../sink/WriteSinkFunctionByMillis.java | 50 +
.../function/source/FileMonitoringFunction.java | 123 ++
.../api/function/source/FileReadFunction.java | 51 +
.../api/function/source/FileSourceFunction.java | 142 ++
.../function/source/FromElementsFunction.java | 49 +
.../function/source/GenSequenceFunction.java | 53 +
.../function/source/GenericSourceFunction.java | 25 +
.../function/source/ParallelSourceFunction.java | 26 +
.../source/RichParallelSourceFunction.java | 27 +
.../api/function/source/RichSourceFunction.java | 27 +
.../source/SocketTextStreamFunction.java | 86 +
.../api/function/source/SourceFunction.java | 29 +
.../api/invokable/ChainableInvokable.java | 39 +
.../streaming/api/invokable/SinkInvokable.java | 50 +
.../api/invokable/SourceInvokable.java | 44 +
.../api/invokable/StreamInvokable.java | 186 +++
.../api/invokable/operator/BatchIterator.java | 25 +
.../invokable/operator/CounterInvokable.java | 43 +
.../api/invokable/operator/FilterInvokable.java | 55 +
.../invokable/operator/FlatMapInvokable.java | 51 +
.../operator/GroupedReduceInvokable.java | 59 +
.../operator/GroupedWindowInvokable.java | 506 ++++++
.../api/invokable/operator/MapInvokable.java | 50 +
.../invokable/operator/ProjectInvokable.java | 61 +
.../operator/StreamReduceInvokable.java | 69 +
.../operator/WindowGroupReduceInvokable.java | 51 +
.../api/invokable/operator/WindowInvokable.java | 382 +++++
.../operator/WindowReduceInvokable.java | 59 +
.../operator/co/CoFlatMapInvokable.java | 54 +
.../operator/co/CoGroupedReduceInvokable.java | 88 ++
.../api/invokable/operator/co/CoInvokable.java | 133 ++
.../invokable/operator/co/CoMapInvokable.java | 54 +
.../operator/co/CoReduceInvokable.java | 70 +
.../operator/co/CoWindowInvokable.java | 197 +++
.../api/streamrecord/StreamRecord.java | 134 ++
.../streamrecord/StreamRecordSerializer.java | 114 ++
.../flink/streaming/api/streamrecord/UID.java | 122 ++
.../api/streamvertex/CoStreamVertex.java | 142 ++
.../api/streamvertex/InputHandler.java | 96 ++
.../api/streamvertex/OutputHandler.java | 242 +++
.../api/streamvertex/StreamIterationHead.java | 101 ++
.../api/streamvertex/StreamIterationTail.java | 103 ++
.../api/streamvertex/StreamTaskContext.java | 40 +
.../api/streamvertex/StreamVertex.java | 156 ++
.../api/streamvertex/StreamVertexException.java | 68 +
.../streamvertex/StreamingRuntimeContext.java | 88 ++
.../windowing/deltafunction/CosineDistance.java | 87 ++
.../windowing/deltafunction/DeltaFunction.java | 44 +
.../deltafunction/EuclideanDistance.java | 58 +
.../ExtractionAwareDeltaFunction.java | 90 ++
.../api/windowing/extractor/ArrayFromTuple.java | 74 +
.../extractor/ConcatinatedExtract.java | 68 +
.../api/windowing/extractor/Extractor.java | 43 +
.../api/windowing/extractor/FieldFromArray.java | 59 +
.../api/windowing/extractor/FieldFromTuple.java | 58 +
.../windowing/extractor/FieldsFromArray.java | 67 +
.../windowing/extractor/FieldsFromTuple.java | 53 +
.../streaming/api/windowing/helper/Count.java | 101 ++
.../streaming/api/windowing/helper/Delta.java | 91 ++
.../api/windowing/helper/SystemTimestamp.java | 37 +
.../streaming/api/windowing/helper/Time.java | 166 ++
.../api/windowing/helper/Timestamp.java | 39 +
.../api/windowing/helper/TimestampWrapper.java | 44 +
.../api/windowing/helper/WindowingHelper.java | 39 +
.../ActiveCloneableEvictionPolicyWrapper.java | 62 +
.../windowing/policy/ActiveEvictionPolicy.java | 52 +
.../policy/ActiveEvictionPolicyWrapper.java | 64 +
.../windowing/policy/ActiveTriggerCallback.java | 45 +
.../windowing/policy/ActiveTriggerPolicy.java | 81 +
.../policy/CloneableEvictionPolicy.java | 45 +
.../policy/CloneableTriggerPolicy.java | 45 +
.../windowing/policy/CountEvictionPolicy.java | 119 ++
.../windowing/policy/CountTriggerPolicy.java | 87 ++
.../api/windowing/policy/DeltaPolicy.java | 112 ++
.../api/windowing/policy/EvictionPolicy.java | 47 +
.../api/windowing/policy/PunctuationPolicy.java | 116 ++
.../windowing/policy/TimeEvictionPolicy.java | 137 ++
.../api/windowing/policy/TimeTriggerPolicy.java | 196 +++
.../api/windowing/policy/TriggerPolicy.java | 54 +
.../policy/TumblingEvictionPolicy.java | 90 ++
.../flink/streaming/io/BlockingQueueBroker.java | 41 +
.../flink/streaming/io/CoReaderIterator.java | 57 +
.../flink/streaming/io/CoRecordReader.java | 221 +++
.../flink/streaming/io/StreamRecordWriter.java | 83 +
.../partitioner/BroadcastPartitioner.java | 55 +
.../partitioner/DistributePartitioner.java | 46 +
.../partitioner/FieldsPartitioner.java | 50 +
.../partitioner/GlobalPartitioner.java | 38 +
.../partitioner/ShufflePartitioner.java | 49 +
.../partitioner/StreamPartitioner.java | 44 +
.../flink/streaming/state/CircularFifoList.java | 112 ++
.../flink/streaming/state/GraphState.java | 61 +
.../apache/flink/streaming/state/MapState.java | 201 +++
.../streaming/state/NullableCircularBuffer.java | 362 +++++
.../flink/streaming/state/OperatorState.java | 105 ++
.../streaming/state/PartitionableState.java | 60 +
.../flink/streaming/state/SimpleState.java | 53 +
.../state/checkpoint/MapCheckpoint.java | 72 +
.../state/checkpoint/StateCheckpoint.java | 82 +
.../flink/streaming/util/ClusterUtil.java | 76 +
.../streaming/util/keys/KeySelectorUtil.java | 126 ++
.../streaming/api/AggregationFunctionTest.java | 318 ++++
.../apache/flink/streaming/api/IterateTest.java | 95 ++
.../flink/streaming/api/OutputSplitterTest.java | 185 +++
.../apache/flink/streaming/api/PrintTest.java | 57 +
.../apache/flink/streaming/api/SourceTest.java | 70 +
.../streaming/api/WindowCrossJoinTest.java | 154 ++
.../api/collector/DirectedOutputTest.java | 112 ++
.../api/collector/OutputSelectorTest.java | 58 +
.../api/collector/StreamCollectorTest.java | 52 +
.../api/invokable/operator/CoFlatMapTest.java | 84 +
.../invokable/operator/CoGroupedReduceTest.java | 125 ++
.../api/invokable/operator/CoMapTest.java | 57 +
.../invokable/operator/CoStreamReduceTest.java | 71 +
.../api/invokable/operator/CoWindowTest.java | 182 +++
.../operator/CounterInvokableTest.java | 39 +
.../api/invokable/operator/FilterTest.java | 51 +
.../api/invokable/operator/FlatMapTest.java | 54 +
.../operator/GroupedReduceInvokableTest.java | 62 +
.../operator/GroupedWindowInvokableTest.java | 574 +++++++
.../api/invokable/operator/MapTest.java | 49 +
.../api/invokable/operator/ProjectTest.java | 67 +
.../invokable/operator/StreamReduceTest.java | 54 +
.../invokable/operator/WindowInvokableTest.java | 261 ++++
.../streaming/api/streamrecord/UIDTest.java | 53 +
.../api/streamvertex/MockRecordWriter.java | 45 +
.../api/streamvertex/StreamVertexTest.java | 177 +++
.../deltafunction/CosineDistanceTest.java | 70 +
.../deltafunction/EuclideanDistanceTest.java | 71 +
.../windowing/extractor/ArrayFromTupleTest.java | 118 ++
.../extractor/ConcatinatedExtractTest.java | 83 +
.../windowing/extractor/FieldFromArrayTest.java | 55 +
.../windowing/extractor/FieldFromTupleTest.java | 84 +
.../extractor/FieldsFromArrayTest.java | 108 ++
.../extractor/FieldsFromTupleTest.java | 106 ++
.../policy/CountEvictionPolicyTest.java | 119 ++
.../policy/CountTriggerPolicyTest.java | 94 ++
.../api/windowing/policy/DeltaPolicyTest.java | 62 +
.../windowing/policy/PunctuationPolicyTest.java | 123 ++
.../policy/TimeEvictionPolicyTest.java | 121 ++
.../windowing/policy/TimeTriggerPolicyTest.java | 114 ++
.../policy/TumblingEvictionPolicyTest.java | 41 +
.../partitioner/BroadcastPartitionerTest.java | 55 +
.../partitioner/DistributePartitionerTest.java | 56 +
.../partitioner/FieldsPartitionerTest.java | 76 +
.../partitioner/ForwardPartitionerTest.java | 55 +
.../partitioner/GlobalPartitionerTest.java | 50 +
.../partitioner/ShufflePartitionerTest.java | 60 +
.../flink/streaming/state/MapStateTest.java | 130 ++
.../streaming/state/OperatorStateTest.java | 48 +
.../flink/streaming/util/MockCoContext.java | 217 +++
.../flink/streaming/util/MockCollector.java | 43 +
.../flink/streaming/util/MockContext.java | 159 ++
.../streaming/util/MockRecordWriterFactory.java | 40 +
.../apache/flink/streaming/util/MockSource.java | 36 +
.../streaming/util/TestStreamEnvironment.java | 72 +
.../src/test/resources/log4j-test.properties | 19 +
.../src/test/resources/logback-test.xml | 30 +
.../flink-streaming-examples/pom.xml | 518 ++++++
.../examples/iteration/IterateExample.java | 168 ++
.../streaming/examples/join/WindowJoin.java | 190 +++
.../ml/IncrementalLearningSkeleton.java | 213 +++
.../socket/SocketTextStreamWordCount.java | 108 ++
.../examples/twitter/TwitterStream.java | 202 +++
.../twitter/util/TwitterStreamData.java | 255 +++
.../examples/windowing/DeltaExtractExample.java | 125 ++
.../windowing/MultiplePoliciesExample.java | 139 ++
.../examples/windowing/SlidingExample.java | 139 ++
.../windowing/TimeWindowingExample.java | 152 ++
.../windowing/TopSpeedWindowingExample.java | 128 ++
.../examples/wordcount/PojoExample.java | 185 +++
.../streaming/examples/wordcount/WordCount.java | 148 ++
.../socket/SocketTextStreamWordCount.scala | 91 ++
.../examples/windowing/TopSpeedWindowing.scala | 94 ++
.../scala/examples/windowing/WindowJoin.scala | 72 +
.../flink-streaming-scala/pom.xml | 217 +++
.../scala/ScalaStreamingAggregator.java | 111 ++
.../api/scala/ConnectedDataStream.scala | 380 +++++
.../flink/streaming/api/scala/DataStream.scala | 573 +++++++
.../streaming/api/scala/SplitDataStream.scala | 39 +
.../api/scala/StreamCrossOperator.scala | 120 ++
.../api/scala/StreamExecutionEnvironment.scala | 289 ++++
.../api/scala/StreamJoinOperator.scala | 212 +++
.../streaming/api/scala/TemporalOperator.scala | 43 +
.../api/scala/WindowedDataStream.scala | 248 +++
.../flink/streaming/api/scala/package.scala | 70 +
.../streaming/api/scala/windowing/Delta.scala | 47 +
.../streaming/api/scala/windowing/Time.scala | 54 +
flink-staging/flink-streaming/pom.xml | 75 +
flink-staging/flink-tachyon/pom.xml | 117 ++
.../java/org/apache/flink/tachyon/HDFSTest.java | 160 ++
.../tachyon/TachyonFileSystemWrapperTest.java | 167 ++
.../src/test/resources/log4j.properties | 23 +
.../src/test/resources/tachyonHadoopConf.xml | 28 +
flink-staging/pom.xml | 64 +
pom.xml | 6 +-
774 files changed, 47728 insertions(+), 47727 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/docs/internal_add_operator.md
----------------------------------------------------------------------
diff --git a/docs/internal_add_operator.md b/docs/internal_add_operator.md
index ddcb6b5..c564221 100644
--- a/docs/internal_add_operator.md
+++ b/docs/internal_add_operator.md
@@ -100,7 +100,7 @@ void setInput(DataSet<IN> inputData);
DataSet<OUT> createResult();
~~~
-The {% gh_link /flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java "VertexCentricIteration" %} operator is implemented that way. Below is an example how to implement the *count()* operator that way.
+The {% gh_link /flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java "VertexCentricIteration" %} operator is implemented that way. Below is an example how to implement the *count()* operator that way.
~~~ java
public class Counter<T> implements CustomUnaryOperation<T, Long> {
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/docs/internal_general_arch.md
----------------------------------------------------------------------
diff --git a/docs/internal_general_arch.md b/docs/internal_general_arch.md
index 81a8cbe..ec07056 100644
--- a/docs/internal_general_arch.md
+++ b/docs/internal_general_arch.md
@@ -75,12 +75,13 @@ In addition to the projects listed in the figure above, Flink currently contains
- `flink-dist`: The *distribution* project. It defines how to assemble the compiled code, scripts, and other resources
into the final folder structure that is ready to use.
-- `flink-addons`: A series of projects that are in an early version. Currently contains
+- `flink-staging`: A series of projects that are in an early version. Currently contains
among other things projects for YARN support, JDBC data sources and sinks, hadoop compatibility,
graph specific operators, and HBase connectors.
- `flink-quickstart`: Scripts, maven archetypes, and example programs for the quickstarts and tutorials.
+- `flink-contrib`: Useful tools contributed by users. The code is maintained mainly by external contributors. The requirements for code being accepted into `flink-contrib` are lower compared to the rest of the code.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/docs/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md
index d2d54c3..3476390 100644
--- a/docs/streaming_guide.md
+++ b/docs/streaming_guide.md
@@ -857,7 +857,7 @@ After installing Docker an image can be pulled for each connector. Containers ca
For the easiest set up create a jar with all the dependencies of the *flink-streaming-connectors* project.
~~~batch
-cd /PATH/TO/GIT/incubator-flink/flink-addons/flink-streaming-connectors
+cd /PATH/TO/GIT/flink/flink-staging/flink-streaming-connectors
mvn assembly:assembly
~~~batch
@@ -974,14 +974,14 @@ Now a terminal started running from the image with all the necessary configurati
To have the latest version of Flink type:
~~~batch
-cd /git/incubator-flink/
+cd /git/flink/
git pull
~~~
Then build the code with:
~~~batch
-cd /git/incubator-flink/flink-addons/flink-streaming/flink-streaming-connectors/
+cd /git/flink/flink-staging/flink-streaming/flink-streaming-connectors/
mvn install -DskipTests
~~~
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/pom.xml b/flink-addons/flink-avro/pom.xml
deleted file mode 100644
index 8115f9d..0000000
--- a/flink-addons/flink-avro/pom.xml
+++ /dev/null
@@ -1,182 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-addons</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-avro</artifactId>
- <name>flink-avro</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <!-- version is derived from base module -->
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>create-test-dependency</id>
- <phase>process-test-classes</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <archive>
- <manifest>
- <mainClass>org.apache.flink.api.avro.testjar.AvroExternalJarProgram</mainClass>
- </manifest>
- </archive>
- <finalName>maven</finalName>
- <attach>false</attach>
- <descriptors>
- <descriptor>src/test/assembly/test-assembly.xml</descriptor>
- </descriptors>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!--Remove the AvroExternalJarProgram code from the test-classes directory since it musn't be in the
- classpath when running the tests to actually test whether the user code class loader
- is properly used.-->
- <plugin>
- <artifactId>maven-clean-plugin</artifactId>
- <version>2.5</version><!--$NO-MVN-MAN-VER$-->
- <executions>
- <execution>
- <id>remove-avroexternalprogram</id>
- <phase>process-test-classes</phase>
- <goals>
- <goal>clean</goal>
- </goals>
- <configuration>
- <excludeDefaultDirectories>true</excludeDefaultDirectories>
- <filesets>
- <fileset>
- <directory>${project.build.testOutputDirectory}</directory>
- <includes>
- <include>**/testjar/*.class</include>
- </includes>
- </fileset>
- </filesets>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- <pluginManagement>
- <plugins>
- <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
- <plugin>
- <groupId>org.eclipse.m2e</groupId>
- <artifactId>lifecycle-mapping</artifactId>
- <version>1.0.0</version>
- <configuration>
- <lifecycleMappingMetadata>
- <pluginExecutions>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <versionRange>[2.4,)</versionRange>
- <goals>
- <goal>single</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore/>
- </action>
- </pluginExecution>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-clean-plugin</artifactId>
- <versionRange>[1,)</versionRange>
- <goals>
- <goal>clean</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore/>
- </action>
- </pluginExecution>
- </pluginExecutions>
- </lifecycleMappingMetadata>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-
- <profiles>
- <profile>
- <!-- A bug with java6 is causing the javadoc generation
- to fail because the test case contains junit?
- See https://github.com/stratosphere/stratosphere/pull/405#issuecomment-32591978
- for further links -->
- <id>disable-javadocs-in-java6</id>
- <activation>
- <jdk>(,1.6]</jdk> <!-- disable javadocs for java6 or lower. -->
- </activation>
- <properties>
- <maven.javadoc.skip>true</maven.javadoc.skip>
- </properties>
- </profile>
- </profiles>
-
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java
deleted file mode 100644
index 59da4cb..0000000
--- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.avro;
-
-import java.io.DataInput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.avro.io.Decoder;
-import org.apache.avro.util.Utf8;
-
-
-public class DataInputDecoder extends Decoder {
-
- private final Utf8 stringDecoder = new Utf8();
-
- private DataInput in;
-
- public void setIn(DataInput in) {
- this.in = in;
- }
-
- // --------------------------------------------------------------------------------------------
- // primitives
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void readNull() {}
-
-
- @Override
- public boolean readBoolean() throws IOException {
- return in.readBoolean();
- }
-
- @Override
- public int readInt() throws IOException {
- return in.readInt();
- }
-
- @Override
- public long readLong() throws IOException {
- return in.readLong();
- }
-
- @Override
- public float readFloat() throws IOException {
- return in.readFloat();
- }
-
- @Override
- public double readDouble() throws IOException {
- return in.readDouble();
- }
-
- @Override
- public int readEnum() throws IOException {
- return readInt();
- }
-
- // --------------------------------------------------------------------------------------------
- // bytes
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void readFixed(byte[] bytes, int start, int length) throws IOException {
- in.readFully(bytes, start, length);
- }
-
- @Override
- public ByteBuffer readBytes(ByteBuffer old) throws IOException {
- int length = readInt();
- ByteBuffer result;
- if (old != null && length <= old.capacity() && old.hasArray()) {
- result = old;
- result.clear();
- } else {
- result = ByteBuffer.allocate(length);
- }
- in.readFully(result.array(), result.arrayOffset() + result.position(), length);
- result.limit(length);
- return result;
- }
-
-
- @Override
- public void skipFixed(int length) throws IOException {
- skipBytes(length);
- }
-
- @Override
- public void skipBytes() throws IOException {
- int num = readInt();
- skipBytes(num);
- }
-
- // --------------------------------------------------------------------------------------------
- // strings
- // --------------------------------------------------------------------------------------------
-
-
- @Override
- public Utf8 readString(Utf8 old) throws IOException {
- int length = readInt();
- Utf8 result = (old != null ? old : new Utf8());
- result.setByteLength(length);
-
- if (length > 0) {
- in.readFully(result.getBytes(), 0, length);
- }
-
- return result;
- }
-
- @Override
- public String readString() throws IOException {
- return readString(stringDecoder).toString();
- }
-
- @Override
- public void skipString() throws IOException {
- int len = readInt();
- skipBytes(len);
- }
-
- // --------------------------------------------------------------------------------------------
- // collection types
- // --------------------------------------------------------------------------------------------
-
- @Override
- public long readArrayStart() throws IOException {
- return readVarLongCount(in);
- }
-
- @Override
- public long arrayNext() throws IOException {
- return readVarLongCount(in);
- }
-
- @Override
- public long skipArray() throws IOException {
- return readVarLongCount(in);
- }
-
- @Override
- public long readMapStart() throws IOException {
- return readVarLongCount(in);
- }
-
- @Override
- public long mapNext() throws IOException {
- return readVarLongCount(in);
- }
-
- @Override
- public long skipMap() throws IOException {
- return readVarLongCount(in);
- }
-
- // --------------------------------------------------------------------------------------------
- // union
- // --------------------------------------------------------------------------------------------
-
- @Override
- public int readIndex() throws IOException {
- return readInt();
- }
-
- // --------------------------------------------------------------------------------------------
- // utils
- // --------------------------------------------------------------------------------------------
-
- private void skipBytes(int num) throws IOException {
- while (num > 0) {
- num -= in.skipBytes(num);
- }
- }
-
- public static long readVarLongCount(DataInput in) throws IOException {
- long value = in.readUnsignedByte();
-
- if ((value & 0x80) == 0) {
- return value;
- }
- else {
- long curr;
- int shift = 7;
- value = value & 0x7f;
- while (((curr = in.readUnsignedByte()) & 0x80) != 0){
- value |= (curr & 0x7f) << shift;
- shift += 7;
- }
- value |= curr << shift;
- return value;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java
deleted file mode 100644
index 0102cc1..0000000
--- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.avro;
-
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.avro.io.Encoder;
-import org.apache.avro.util.Utf8;
-
-
-public final class DataOutputEncoder extends Encoder implements java.io.Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private DataOutput out;
-
-
- public void setOut(DataOutput out) {
- this.out = out;
- }
-
-
- @Override
- public void flush() throws IOException {}
-
- // --------------------------------------------------------------------------------------------
- // primitives
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void writeNull() {}
-
-
- @Override
- public void writeBoolean(boolean b) throws IOException {
- out.writeBoolean(b);
- }
-
- @Override
- public void writeInt(int n) throws IOException {
- out.writeInt(n);
- }
-
- @Override
- public void writeLong(long n) throws IOException {
- out.writeLong(n);
- }
-
- @Override
- public void writeFloat(float f) throws IOException {
- out.writeFloat(f);
- }
-
- @Override
- public void writeDouble(double d) throws IOException {
- out.writeDouble(d);
- }
-
- @Override
- public void writeEnum(int e) throws IOException {
- out.writeInt(e);
- }
-
-
- // --------------------------------------------------------------------------------------------
- // bytes
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void writeFixed(byte[] bytes, int start, int len) throws IOException {
- out.write(bytes, start, len);
- }
-
- @Override
- public void writeBytes(byte[] bytes, int start, int len) throws IOException {
- out.writeInt(len);
- if (len > 0) {
- out.write(bytes, start, len);
- }
- }
-
- @Override
- public void writeBytes(ByteBuffer bytes) throws IOException {
- int num = bytes.remaining();
- out.writeInt(num);
-
- if (num > 0) {
- writeFixed(bytes);
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // strings
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void writeString(String str) throws IOException {
- byte[] bytes = Utf8.getBytesFor(str);
- writeBytes(bytes, 0, bytes.length);
- }
-
- @Override
- public void writeString(Utf8 utf8) throws IOException {
- writeBytes(utf8.getBytes(), 0, utf8.getByteLength());
-
- }
-
- // --------------------------------------------------------------------------------------------
- // collection types
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void writeArrayStart() {}
-
- @Override
- public void setItemCount(long itemCount) throws IOException {
- if (itemCount > 0) {
- writeVarLongCount(out, itemCount);
- }
- }
-
- @Override
- public void startItem() {}
-
- @Override
- public void writeArrayEnd() throws IOException {
- // write a single byte 0, shortcut for a var-length long of 0
- out.write(0);
- }
-
- @Override
- public void writeMapStart() {}
-
- @Override
- public void writeMapEnd() throws IOException {
- // write a single byte 0, shortcut for a var-length long of 0
- out.write(0);
- }
-
- // --------------------------------------------------------------------------------------------
- // union
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void writeIndex(int unionIndex) throws IOException {
- out.writeInt(unionIndex);
- }
-
- // --------------------------------------------------------------------------------------------
- // utils
- // --------------------------------------------------------------------------------------------
-
-
- public static void writeVarLongCount(DataOutput out, long val) throws IOException {
- if (val < 0) {
- throw new IOException("Illegal count (must be non-negative): " + val);
- }
-
- while ((val & ~0x7FL) != 0) {
- out.write(((int) val) | 0x80);
- val >>>= 7;
- }
- out.write((int) val);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java
deleted file mode 100644
index 655b6ff..0000000
--- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.avro;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-import org.apache.avro.file.SeekableInput;
-import org.apache.flink.core.fs.FSDataInputStream;
-
-
-/**
- * Code copy pasted from org.apache.avro.mapred.FSInput (which is Apache licensed as well)
- *
- * The wrapper keeps track of the position in the data stream.
- */
-public class FSDataInputStreamWrapper implements Closeable, SeekableInput {
- private final FSDataInputStream stream;
- private final long len;
- private long pos;
-
- public FSDataInputStreamWrapper(FSDataInputStream stream, long len) {
- this.stream = stream;
- this.len = len;
- this.pos = 0;
- }
-
- public long length() {
- return len;
- }
-
- public int read(byte[] b, int off, int len) throws IOException {
- int read;
- read = stream.read(b, off, len);
- pos += read;
- return read;
- }
-
- public void seek(long p) throws IOException {
- stream.seek(p);
- pos = p;
- }
-
- public long tell() throws IOException {
- return pos;
- }
-
- public void close() throws IOException {
- stream.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java
deleted file mode 100644
index 6affeec..0000000
--- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.io.avro.example;
-
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.io.GenericInputFormat;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.Collector;
-
-@SuppressWarnings("serial")
-public class AvroTypeExample {
-
-
- public static void main(String[] args) throws Exception {
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<User> users = env.createInput(new UserGeneratingInputFormat());
-
- users
- .map(new NumberExtractingMapper())
- .groupBy(1)
- .reduceGroup(new ConcatenatingReducer())
- .print();
-
- env.execute();
- }
-
-
-
- public static final class NumberExtractingMapper implements MapFunction<User, Tuple2<User, Integer>> {
-
- @Override
- public Tuple2<User, Integer> map(User user) {
- return new Tuple2<User, Integer>(user, user.getFavoriteNumber());
- }
- }
-
-
- public static final class ConcatenatingReducer implements GroupReduceFunction<Tuple2<User, Integer>, Tuple2<Integer, String>> {
-
- @Override
- public void reduce(Iterable<Tuple2<User, Integer>> values, Collector<Tuple2<Integer, String>> out) throws Exception {
- int number = 0;
- StringBuilder colors = new StringBuilder();
-
- for (Tuple2<User, Integer> u : values) {
- number = u.f1;
- colors.append(u.f0.getFavoriteColor()).append(" - ");
- }
-
- colors.setLength(colors.length() - 3);
- out.collect(new Tuple2<Integer, String>(number, colors.toString()));
- }
- }
-
-
- public static final class UserGeneratingInputFormat extends GenericInputFormat<User> {
-
- private static final long serialVersionUID = 1L;
-
- private static final int NUM = 100;
-
- private final Random rnd = new Random(32498562304986L);
-
- private static final String[] NAMES = { "Peter", "Bob", "Liddy", "Alexander", "Stan" };
-
- private static final String[] COLORS = { "mauve", "crimson", "copper", "sky", "grass" };
-
- private int count;
-
-
- @Override
- public boolean reachedEnd() throws IOException {
- return count >= NUM;
- }
-
- @Override
- public User nextRecord(User reuse) throws IOException {
- count++;
-
- User u = new User();
- u.setName(NAMES[rnd.nextInt(NAMES.length)]);
- u.setFavoriteColor(COLORS[rnd.nextInt(COLORS.length)]);
- u.setFavoriteNumber(rnd.nextInt(87));
- return u;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java
deleted file mode 100644
index 4608f96..0000000
--- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/**
- * Autogenerated by Avro
- *
- * DO NOT EDIT DIRECTLY
- */
-package org.apache.flink.api.io.avro.example;
-@SuppressWarnings("all")
-@org.apache.avro.specific.AvroGenerated
-public class User extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"org.apache.flink.api.avro.example\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}");
- public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
- @Deprecated public java.lang.CharSequence name;
- @Deprecated public java.lang.Integer favorite_number;
- @Deprecated public java.lang.CharSequence favorite_color;
-
- /**
- * Default constructor. Note that this does not initialize fields
- * to their default values from the schema. If that is desired then
- * one should use {@link #newBuilder()}.
- */
- public User() {}
-
- /**
- * All-args constructor.
- */
- public User(java.lang.CharSequence name, java.lang.Integer favorite_number, java.lang.CharSequence favorite_color) {
- this.name = name;
- this.favorite_number = favorite_number;
- this.favorite_color = favorite_color;
- }
-
- public org.apache.avro.Schema getSchema() { return SCHEMA$; }
- // Used by DatumWriter. Applications should not call.
- public java.lang.Object get(int field$) {
- switch (field$) {
- case 0: return name;
- case 1: return favorite_number;
- case 2: return favorite_color;
- default: throw new org.apache.avro.AvroRuntimeException("Bad index");
- }
- }
- // Used by DatumReader. Applications should not call.
- @SuppressWarnings(value="unchecked")
- public void put(int field$, java.lang.Object value$) {
- switch (field$) {
- case 0: name = (java.lang.CharSequence)value$; break;
- case 1: favorite_number = (java.lang.Integer)value$; break;
- case 2: favorite_color = (java.lang.CharSequence)value$; break;
- default: throw new org.apache.avro.AvroRuntimeException("Bad index");
- }
- }
-
- /**
- * Gets the value of the 'name' field.
- */
- public java.lang.CharSequence getName() {
- return name;
- }
-
- /**
- * Sets the value of the 'name' field.
- * @param value the value to set.
- */
- public void setName(java.lang.CharSequence value) {
- this.name = value;
- }
-
- /**
- * Gets the value of the 'favorite_number' field.
- */
- public java.lang.Integer getFavoriteNumber() {
- return favorite_number;
- }
-
- /**
- * Sets the value of the 'favorite_number' field.
- * @param value the value to set.
- */
- public void setFavoriteNumber(java.lang.Integer value) {
- this.favorite_number = value;
- }
-
- /**
- * Gets the value of the 'favorite_color' field.
- */
- public java.lang.CharSequence getFavoriteColor() {
- return favorite_color;
- }
-
- /**
- * Sets the value of the 'favorite_color' field.
- * @param value the value to set.
- */
- public void setFavoriteColor(java.lang.CharSequence value) {
- this.favorite_color = value;
- }
-
- /** Creates a new User RecordBuilder */
- public static org.apache.flink.api.io.avro.example.User.Builder newBuilder() {
- return new org.apache.flink.api.io.avro.example.User.Builder();
- }
-
- /** Creates a new User RecordBuilder by copying an existing Builder */
- public static org.apache.flink.api.io.avro.example.User.Builder newBuilder(org.apache.flink.api.io.avro.example.User.Builder other) {
- return new org.apache.flink.api.io.avro.example.User.Builder(other);
- }
-
- /** Creates a new User RecordBuilder by copying an existing User instance */
- public static org.apache.flink.api.io.avro.example.User.Builder newBuilder(org.apache.flink.api.io.avro.example.User other) {
- return new org.apache.flink.api.io.avro.example.User.Builder(other);
- }
-
- /**
- * RecordBuilder for User instances.
- */
- public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<User>
- implements org.apache.avro.data.RecordBuilder<User> {
-
- private java.lang.CharSequence name;
- private java.lang.Integer favorite_number;
- private java.lang.CharSequence favorite_color;
-
- /** Creates a new Builder */
- private Builder() {
- super(org.apache.flink.api.io.avro.example.User.SCHEMA$);
- }
-
- /** Creates a Builder by copying an existing Builder */
- private Builder(org.apache.flink.api.io.avro.example.User.Builder other) {
- super(other);
- if (isValidValue(fields()[0], other.name)) {
- this.name = data().deepCopy(fields()[0].schema(), other.name);
- fieldSetFlags()[0] = true;
- }
- if (isValidValue(fields()[1], other.favorite_number)) {
- this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);
- fieldSetFlags()[1] = true;
- }
- if (isValidValue(fields()[2], other.favorite_color)) {
- this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);
- fieldSetFlags()[2] = true;
- }
- }
-
- /** Creates a Builder by copying an existing User instance */
- private Builder(org.apache.flink.api.io.avro.example.User other) {
- super(org.apache.flink.api.io.avro.example.User.SCHEMA$);
- if (isValidValue(fields()[0], other.name)) {
- this.name = data().deepCopy(fields()[0].schema(), other.name);
- fieldSetFlags()[0] = true;
- }
- if (isValidValue(fields()[1], other.favorite_number)) {
- this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);
- fieldSetFlags()[1] = true;
- }
- if (isValidValue(fields()[2], other.favorite_color)) {
- this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);
- fieldSetFlags()[2] = true;
- }
- }
-
- /** Gets the value of the 'name' field */
- public java.lang.CharSequence getName() {
- return name;
- }
-
- /** Sets the value of the 'name' field */
- public org.apache.flink.api.io.avro.example.User.Builder setName(java.lang.CharSequence value) {
- validate(fields()[0], value);
- this.name = value;
- fieldSetFlags()[0] = true;
- return this;
- }
-
- /** Checks whether the 'name' field has been set */
- public boolean hasName() {
- return fieldSetFlags()[0];
- }
-
- /** Clears the value of the 'name' field */
- public org.apache.flink.api.io.avro.example.User.Builder clearName() {
- name = null;
- fieldSetFlags()[0] = false;
- return this;
- }
-
- /** Gets the value of the 'favorite_number' field */
- public java.lang.Integer getFavoriteNumber() {
- return favorite_number;
- }
-
- /** Sets the value of the 'favorite_number' field */
- public org.apache.flink.api.io.avro.example.User.Builder setFavoriteNumber(java.lang.Integer value) {
- validate(fields()[1], value);
- this.favorite_number = value;
- fieldSetFlags()[1] = true;
- return this;
- }
-
- /** Checks whether the 'favorite_number' field has been set */
- public boolean hasFavoriteNumber() {
- return fieldSetFlags()[1];
- }
-
- /** Clears the value of the 'favorite_number' field */
- public org.apache.flink.api.io.avro.example.User.Builder clearFavoriteNumber() {
- favorite_number = null;
- fieldSetFlags()[1] = false;
- return this;
- }
-
- /** Gets the value of the 'favorite_color' field */
- public java.lang.CharSequence getFavoriteColor() {
- return favorite_color;
- }
-
- /** Sets the value of the 'favorite_color' field */
- public org.apache.flink.api.io.avro.example.User.Builder setFavoriteColor(java.lang.CharSequence value) {
- validate(fields()[2], value);
- this.favorite_color = value;
- fieldSetFlags()[2] = true;
- return this;
- }
-
- /** Checks whether the 'favorite_color' field has been set */
- public boolean hasFavoriteColor() {
- return fieldSetFlags()[2];
- }
-
- /** Clears the value of the 'favorite_color' field */
- public org.apache.flink.api.io.avro.example.User.Builder clearFavoriteColor() {
- favorite_color = null;
- fieldSetFlags()[2] = false;
- return this;
- }
-
- @Override
- public User build() {
- try {
- User record = new User();
- record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]);
- record.favorite_number = fieldSetFlags()[1] ? this.favorite_number : (java.lang.Integer) defaultValue(fields()[1]);
- record.favorite_color = fieldSetFlags()[2] ? this.favorite_color : (java.lang.CharSequence) defaultValue(fields()[2]);
- return record;
- } catch (Exception e) {
- throw new org.apache.avro.AvroRuntimeException(e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java
deleted file mode 100644
index fbb0f4c..0000000
--- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.java.io;
-
-import java.io.IOException;
-
-import org.apache.avro.file.DataFileReader;
-import org.apache.avro.file.FileReader;
-import org.apache.avro.file.SeekableInput;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.reflect.ReflectDatumReader;
-import org.apache.avro.specific.SpecificDatumReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.flink.api.avro.FSDataInputStreamWrapper;
-import org.apache.flink.api.common.io.FileInputFormat;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.core.fs.FileInputSplit;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.util.InstantiationUtil;
-
-
-public class AvroInputFormat<E> extends FileInputFormat<E> implements ResultTypeQueryable<E> {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(AvroInputFormat.class);
-
-
- private final Class<E> avroValueType;
-
- private boolean reuseAvroValue = true;
-
-
- private transient FileReader<E> dataFileReader;
-
-
- public AvroInputFormat(Path filePath, Class<E> type) {
- super(filePath);
- this.avroValueType = type;
- this.unsplittable = true;
- }
-
-
- /**
- * Sets the flag whether to reuse the Avro value instance for all records.
- * By default, the input format reuses the Avro value.
- *
- * @param reuseAvroValue True, if the input format should reuse the Avro value instance, false otherwise.
- */
- public void setReuseAvroValue(boolean reuseAvroValue) {
- this.reuseAvroValue = reuseAvroValue;
- }
-
- // --------------------------------------------------------------------------------------------
- // Typing
- // --------------------------------------------------------------------------------------------
-
- @Override
- public TypeInformation<E> getProducedType() {
- return TypeExtractor.getForClass(this.avroValueType);
- }
-
- // --------------------------------------------------------------------------------------------
- // Input Format Methods
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void open(FileInputSplit split) throws IOException {
- super.open(split);
-
- DatumReader<E> datumReader;
- if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) {
- datumReader = new SpecificDatumReader<E>(avroValueType);
- } else {
- datumReader = new ReflectDatumReader<E>(avroValueType);
- }
-
- LOG.info("Opening split " + split);
-
- SeekableInput in = new FSDataInputStreamWrapper(stream, (int) split.getLength());
-
- dataFileReader = DataFileReader.openReader(in, datumReader);
- dataFileReader.sync(split.getStart());
- }
-
- @Override
- public boolean reachedEnd() throws IOException {
- return !dataFileReader.hasNext();
- }
-
- @Override
- public E nextRecord(E reuseValue) throws IOException {
- if (!dataFileReader.hasNext()) {
- return null;
- }
-
- if (!reuseAvroValue) {
- reuseValue = InstantiationUtil.instantiate(avroValueType, Object.class);
- }
-
- reuseValue = dataFileReader.next(reuseValue);
- return reuseValue;
- }
-}
[17/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
new file mode 100644
index 0000000..24ebf46
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.ScheduleMode;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
+import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
+import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.streaming.partitioner.StreamPartitioner.PartitioningStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StreamingJobGraphGenerator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StreamingJobGraphGenerator.class);
+
+ private StreamGraph streamGraph;
+
+ private Map<String, AbstractJobVertex> streamVertices;
+ private JobGraph jobGraph;
+ private Collection<String> builtNodes;
+
+ private Map<String, Map<String, StreamConfig>> chainedConfigs;
+ private Map<String, StreamConfig> vertexConfigs;
+ private Map<String, String> chainedNames;
+
+ public StreamingJobGraphGenerator(StreamGraph streamGraph) {
+ this.streamGraph = streamGraph;
+ }
+
+ private void init() {
+ this.streamVertices = new HashMap<String, AbstractJobVertex>();
+ this.builtNodes = new HashSet<String>();
+ this.chainedConfigs = new HashMap<String, Map<String, StreamConfig>>();
+ this.vertexConfigs = new HashMap<String, StreamConfig>();
+ this.chainedNames = new HashMap<String, String>();
+ }
+
+ public JobGraph createJobGraph(String jobName) {
+ jobGraph = new JobGraph(jobName);
+ // Turn lazy scheduling off
+ jobGraph.setScheduleMode(ScheduleMode.ALL);
+
+ init();
+
+ for (String sourceName : streamGraph.getSources()) {
+ createChain(sourceName, sourceName);
+ }
+
+ setSlotSharing();
+
+ return jobGraph;
+ }
+
+ private List<Tuple2<String, String>> createChain(String startNode, String current) {
+
+ if (!builtNodes.contains(startNode)) {
+
+ List<Tuple2<String, String>> transitiveOutEdges = new ArrayList<Tuple2<String, String>>();
+ List<String> chainableOutputs = new ArrayList<String>();
+ List<String> nonChainableOutputs = new ArrayList<String>();
+
+ for (String outName : streamGraph.getOutEdges(current)) {
+ if (isChainable(current, outName)) {
+ chainableOutputs.add(outName);
+ } else {
+ nonChainableOutputs.add(outName);
+ }
+ }
+
+ for (String chainable : chainableOutputs) {
+ transitiveOutEdges.addAll(createChain(startNode, chainable));
+ }
+
+ for (String nonChainable : nonChainableOutputs) {
+ transitiveOutEdges.add(new Tuple2<String, String>(current, nonChainable));
+ createChain(nonChainable, nonChainable);
+ }
+
+ chainedNames.put(current, createChainedName(current, chainableOutputs));
+
+ StreamConfig config = current.equals(startNode) ? createProcessingVertex(startNode)
+ : new StreamConfig(new Configuration());
+
+ setVertexConfig(current, config, chainableOutputs, nonChainableOutputs);
+
+ if (current.equals(startNode)) {
+
+ config.setChainStart();
+ config.setOutEdgesInOrder(transitiveOutEdges);
+
+ for (Tuple2<String, String> edge : transitiveOutEdges) {
+ connect(startNode, edge);
+ }
+
+ config.setTransitiveChainedTaskConfigs(chainedConfigs.get(startNode));
+
+ } else {
+
+ Map<String, StreamConfig> chainedConfs = chainedConfigs.get(startNode);
+
+ if (chainedConfs == null) {
+ chainedConfigs.put(startNode, new HashMap<String, StreamConfig>());
+ }
+ chainedConfigs.get(startNode).put(current, config);
+ }
+
+ return transitiveOutEdges;
+
+ } else {
+ return new ArrayList<Tuple2<String, String>>();
+ }
+ }
+
+ private String createChainedName(String vertexID, List<String> chainedOutputs) {
+ String vertexName = streamGraph.getOperatorName(vertexID);
+ if (chainedOutputs.size() > 1) {
+ List<String> outputChainedNames = new ArrayList<String>();
+ for (String chainable : chainedOutputs) {
+ outputChainedNames.add(chainedNames.get(chainable));
+ }
+ return vertexName + " -> (" + StringUtils.join(outputChainedNames, ", ") + ")";
+ } else if (chainedOutputs.size() == 1) {
+ return vertexName + " -> " + chainedNames.get(chainedOutputs.get(0));
+ } else {
+ return vertexName;
+ }
+
+ }
+
+ private StreamConfig createProcessingVertex(String vertexName) {
+
+ AbstractJobVertex vertex = new AbstractJobVertex(chainedNames.get(vertexName));
+
+ vertex.setInvokableClass(streamGraph.getJobVertexClass(vertexName));
+ if (streamGraph.getParallelism(vertexName) > 0) {
+ vertex.setParallelism(streamGraph.getParallelism(vertexName));
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Parallelism set: {} for {}", streamGraph.getParallelism(vertexName),
+ vertexName);
+ }
+
+ if (streamGraph.getInputFormat(vertexName) != null) {
+ vertex.setInputSplitSource(streamGraph.getInputFormat(vertexName));
+ }
+
+ streamVertices.put(vertexName, vertex);
+ builtNodes.add(vertexName);
+ jobGraph.addVertex(vertex);
+
+ return new StreamConfig(vertex.getConfiguration());
+ }
+
+ private void setVertexConfig(String vertexName, StreamConfig config,
+ List<String> chainableOutputs, List<String> nonChainableOutputs) {
+
+ config.setVertexName(vertexName);
+ config.setBufferTimeout(streamGraph.getBufferTimeout(vertexName));
+
+ config.setTypeSerializerIn1(streamGraph.getInSerializer1(vertexName));
+ config.setTypeSerializerIn2(streamGraph.getInSerializer2(vertexName));
+ config.setTypeSerializerOut1(streamGraph.getOutSerializer1(vertexName));
+ config.setTypeSerializerOut2(streamGraph.getOutSerializer2(vertexName));
+
+ config.setUserInvokable(streamGraph.getInvokable(vertexName));
+ config.setOutputSelectors(streamGraph.getOutputSelector(vertexName));
+ config.setOperatorStates(streamGraph.getState(vertexName));
+
+ config.setNumberOfOutputs(nonChainableOutputs.size());
+ config.setOutputs(nonChainableOutputs);
+ config.setChainedOutputs(chainableOutputs);
+
+ Class<? extends AbstractInvokable> vertexClass = streamGraph.getJobVertexClass(vertexName);
+
+ if (vertexClass.equals(StreamIterationHead.class)
+ || vertexClass.equals(StreamIterationTail.class)) {
+ config.setIterationId(streamGraph.getIterationID(vertexName));
+ config.setIterationWaitTime(streamGraph.getIterationTimeout(vertexName));
+ }
+
+ List<String> allOutputs = new ArrayList<String>(chainableOutputs);
+ allOutputs.addAll(nonChainableOutputs);
+
+ for (String output : allOutputs) {
+ config.setSelectedNames(output, streamGraph.getSelectedNames(vertexName, output));
+ }
+
+ vertexConfigs.put(vertexName, config);
+ }
+
+ private <T> void connect(String headOfChain, Tuple2<String, String> edge) {
+
+ String upStreamVertexName = edge.f0;
+ String downStreamVertexName = edge.f1;
+
+ int outputIndex = streamGraph.getOutEdges(upStreamVertexName).indexOf(downStreamVertexName);
+
+ AbstractJobVertex headVertex = streamVertices.get(headOfChain);
+ AbstractJobVertex downStreamVertex = streamVertices.get(downStreamVertexName);
+
+ StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());
+ StreamConfig upStreamConfig = new StreamConfig(headVertex.getConfiguration());
+
+ List<Integer> outEdgeIndexList = streamGraph.getOutEdgeTypes(upStreamVertexName);
+ int numOfInputs = downStreamConfig.getNumberOfInputs();
+
+ downStreamConfig.setInputIndex(numOfInputs++, outEdgeIndexList.get(outputIndex));
+ downStreamConfig.setNumberOfInputs(numOfInputs);
+
+ StreamPartitioner<?> partitioner = streamGraph.getOutPartitioner(upStreamVertexName,
+ downStreamVertexName);
+
+ upStreamConfig.setPartitioner(downStreamVertexName, partitioner);
+
+ if (partitioner.getStrategy() == PartitioningStrategy.FORWARD) {
+ downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
+ } else {
+ downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.ALL_TO_ALL);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(),
+ headOfChain, downStreamVertexName);
+ }
+ }
+
+ private boolean isChainable(String vertexName, String outName) {
+
+ StreamInvokable<?, ?> headInvokable = streamGraph.getInvokable(vertexName);
+ StreamInvokable<?, ?> outInvokable = streamGraph.getInvokable(outName);
+
+ return streamGraph.getInEdges(outName).size() == 1
+ && outInvokable != null
+ && outInvokable.getChainingStrategy() == ChainingStrategy.ALWAYS
+ && (headInvokable.getChainingStrategy() == ChainingStrategy.HEAD || headInvokable
+ .getChainingStrategy() == ChainingStrategy.ALWAYS)
+ && streamGraph.getOutPartitioner(vertexName, outName).getStrategy() == PartitioningStrategy.FORWARD
+ && streamGraph.getParallelism(vertexName) == streamGraph.getParallelism(outName)
+ && streamGraph.chaining;
+ }
+
+ private void setSlotSharing() {
+ SlotSharingGroup shareGroup = new SlotSharingGroup();
+
+ for (AbstractJobVertex vertex : streamVertices.values()) {
+ vertex.setSlotSharingGroup(shareGroup);
+ }
+
+ for (Integer iterID : streamGraph.getIterationIDs()) {
+ CoLocationGroup ccg = new CoLocationGroup();
+ AbstractJobVertex tail = streamVertices.get(streamGraph.getIterationTail(iterID));
+ AbstractJobVertex head = streamVertices.get(streamGraph.getIterationHead(iterID));
+
+ ccg.addVertex(head);
+ ccg.addVertex(tail);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
new file mode 100644
index 0000000..1281bf0
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.util.Collector;
+
+public class CollectorWrapper<OUT> implements Collector<OUT> {
+
+ private List<Collector<OUT>> outputs;
+
+ public CollectorWrapper() {
+ this.outputs = new LinkedList<Collector<OUT>>();
+ }
+
+ @SuppressWarnings("unchecked")
+ public void addCollector(Collector<?> output) {
+ outputs.add((Collector<OUT>) output);
+ }
+
+ @Override
+ public void collect(OUT record) {
+ for(Collector<OUT> output: outputs){
+ output.collect(record);
+ }
+ }
+
+ @Override
+ public void close() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
new file mode 100755
index 0000000..4681cd3
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A StreamCollector that uses user defined output names and a user defined
+ * output selector to make directed emits.
+ *
+ * @param <OUT>
+ * Type of the Tuple collected.
+ */
+public class DirectedCollectorWrapper<OUT> extends CollectorWrapper<OUT> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DirectedCollectorWrapper.class);
+
+ List<OutputSelector<OUT>> outputSelectors;
+
+ protected Map<String, List<Collector<OUT>>> outputMap;
+
+ private List<Collector<OUT>> selectAllOutputs;
+ private Set<Collector<OUT>> emitted;
+
+ /**
+ * Creates a new DirectedStreamCollector
+ *
+ * @param outputSelector
+ * User defined {@link OutputSelector}
+ */
+ public DirectedCollectorWrapper(List<OutputSelector<OUT>> outputSelectors) {
+ this.outputSelectors = outputSelectors;
+ this.emitted = new HashSet<Collector<OUT>>();
+ this.selectAllOutputs = new LinkedList<Collector<OUT>>();
+ this.outputMap = new HashMap<String, List<Collector<OUT>>>();
+
+ }
+
+ @Override
+ public void addCollector(Collector<?> output) {
+ addCollector(output, new ArrayList<String>());
+ }
+
+ @SuppressWarnings("unchecked")
+ public void addCollector(Collector<?> output, List<String> selectedNames) {
+
+ if (selectedNames.isEmpty()) {
+ selectAllOutputs.add((Collector<OUT>) output);
+ } else {
+ for (String selectedName : selectedNames) {
+
+ if (!outputMap.containsKey(selectedName)) {
+ outputMap.put(selectedName, new LinkedList<Collector<OUT>>());
+ outputMap.get(selectedName).add((Collector<OUT>) output);
+ } else {
+ if (!outputMap.get(selectedName).contains(output)) {
+ outputMap.get(selectedName).add((Collector<OUT>) output);
+ }
+ }
+
+ }
+ }
+ }
+
+ @Override
+ public void collect(OUT record) {
+ emitted.clear();
+
+ for (Collector<OUT> output : selectAllOutputs) {
+ output.collect(record);
+ emitted.add(output);
+ }
+
+ for (OutputSelector<OUT> outputSelector : outputSelectors) {
+ Iterable<String> outputNames = outputSelector.select(record);
+
+ for (String outputName : outputNames) {
+ List<Collector<OUT>> outputList = outputMap.get(outputName);
+ if (outputList == null) {
+ if (LOG.isErrorEnabled()) {
+ String format = String.format(
+ "Cannot emit because no output is selected with the name: %s",
+ outputName);
+ LOG.error(format);
+
+ }
+ } else {
+ for (Collector<OUT> output : outputList) {
+ if (!emitted.contains(output)) {
+ output.collect(record);
+ emitted.add(output);
+ }
+ }
+
+ }
+
+ }
+ }
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/OutputSelector.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/OutputSelector.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/OutputSelector.java
new file mode 100644
index 0000000..6dbcff4
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/OutputSelector.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.io.Serializable;
+
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.datastream.SplitDataStream;
+
+/**
+ * Interface for defining an OutputSelector for a {@link SplitDataStream} using
+ * the {@link SingleOutputStreamOperator#split} call. Every output object of a
+ * {@link SplitDataStream} will run through this operator to select outputs.
+ *
+ * @param <OUT>
+ * Type parameter of the split values.
+ */
+public interface OutputSelector<OUT> extends Serializable {
+ /**
+ * Method for selecting output names for the emitted objects when using the
+ * {@link SingleOutputStreamOperator#split} method. The values will be
+ * emitted only to output names which are contained in the returned
+ * iterable.
+ *
+ * @param value
+ * Output object for which the output selection should be made.
+ */
+ public Iterable<String> select(OUT value);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
new file mode 100644
index 0000000..4551c5a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.io.IOException;
+
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.io.StreamRecordWriter;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StreamOutput<OUT> implements Collector<OUT> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StreamOutput.class);
+
+ private RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
+ private SerializationDelegate<StreamRecord<OUT>> serializationDelegate;
+ private StreamRecord<OUT> streamRecord;
+ private int channelID;
+
+ public StreamOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
+ int channelID, SerializationDelegate<StreamRecord<OUT>> serializationDelegate) {
+
+ this.serializationDelegate = serializationDelegate;
+
+ if (serializationDelegate != null) {
+ this.streamRecord = serializationDelegate.getInstance();
+ } else {
+ throw new RuntimeException("Serializer cannot be null");
+ }
+ this.channelID = channelID;
+ this.output = output;
+ }
+
+ public RecordWriter<SerializationDelegate<StreamRecord<OUT>>> getRecordWriter() {
+ return output;
+ }
+
+ @Override
+ public void collect(OUT record) {
+ streamRecord.setObject(record);
+ streamRecord.newId(channelID);
+ serializationDelegate.setInstance(streamRecord);
+
+ try {
+ output.emit(serializationDelegate);
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Emit failed due to: {}", StringUtils.stringifyException(e));
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ if (output instanceof StreamRecordWriter) {
+ ((StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>) output).close();
+ } else {
+ try {
+ output.flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
new file mode 100755
index 0000000..a6eade8
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -0,0 +1,419 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.ClosureCleaner;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.StreamGraph;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
+import org.apache.flink.streaming.api.function.co.CoMapFunction;
+import org.apache.flink.streaming.api.function.co.CoReduceFunction;
+import org.apache.flink.streaming.api.function.co.CoWindowFunction;
+import org.apache.flink.streaming.api.function.co.RichCoMapFunction;
+import org.apache.flink.streaming.api.function.co.RichCoReduceFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoFlatMapInvokable;
+import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedReduceInvokable;
+import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
+import org.apache.flink.streaming.api.invokable.operator.co.CoMapInvokable;
+import org.apache.flink.streaming.api.invokable.operator.co.CoReduceInvokable;
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
+import org.apache.flink.streaming.api.windowing.helper.SystemTimestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+
+/**
+ * The ConnectedDataStream represents a stream for two different data types. It
+ * can be used to apply transformations like {@link CoMapFunction} on two
+ * {@link DataStream}s
+ *
+ * @param <IN1>
+ * Type of the first input data steam.
+ * @param <IN2>
+ * Type of the second input data stream.
+ */
+public class ConnectedDataStream<IN1, IN2> {
+
+ protected StreamExecutionEnvironment environment;
+ protected StreamGraph jobGraphBuilder;
+ protected DataStream<IN1> dataStream1;
+ protected DataStream<IN2> dataStream2;
+
+ protected boolean isGrouped;
+ protected KeySelector<IN1, ?> keySelector1;
+ protected KeySelector<IN2, ?> keySelector2;
+
+ protected ConnectedDataStream(DataStream<IN1> input1, DataStream<IN2> input2) {
+ this.jobGraphBuilder = input1.streamGraph;
+ this.environment = input1.environment;
+ this.dataStream1 = input1.copy();
+ this.dataStream2 = input2.copy();
+
+ if ((input1 instanceof GroupedDataStream) && (input2 instanceof GroupedDataStream)) {
+ this.isGrouped = true;
+ this.keySelector1 = ((GroupedDataStream<IN1>) input1).keySelector;
+ this.keySelector2 = ((GroupedDataStream<IN2>) input2).keySelector;
+ } else {
+ this.isGrouped = false;
+ this.keySelector1 = null;
+ this.keySelector2 = null;
+ }
+ }
+
+ protected ConnectedDataStream(ConnectedDataStream<IN1, IN2> coDataStream) {
+ this.jobGraphBuilder = coDataStream.jobGraphBuilder;
+ this.environment = coDataStream.environment;
+ this.dataStream1 = coDataStream.getFirst();
+ this.dataStream2 = coDataStream.getSecond();
+ this.isGrouped = coDataStream.isGrouped;
+ this.keySelector1 = coDataStream.keySelector1;
+ this.keySelector2 = coDataStream.keySelector2;
+ }
+
+ public <F> F clean(F f) {
+ if (getExecutionEnvironment().getConfig().isClosureCleanerEnabled()) {
+ ClosureCleaner.clean(f, true);
+ }
+ ClosureCleaner.ensureSerializable(f);
+ return f;
+ }
+
+ public StreamExecutionEnvironment getExecutionEnvironment() {
+ return environment;
+ }
+
+ /**
+ * Returns the first {@link DataStream}.
+ *
+ * @return The first DataStream.
+ */
+ public DataStream<IN1> getFirst() {
+ return dataStream1.copy();
+ }
+
+ /**
+ * Returns the second {@link DataStream}.
+ *
+ * @return The second DataStream.
+ */
+ public DataStream<IN2> getSecond() {
+ return dataStream2.copy();
+ }
+
+ /**
+ * Gets the type of the first input
+ *
+ * @return The type of the first input
+ */
+ public TypeInformation<IN1> getInputType1() {
+ return dataStream1.getType();
+ }
+
+ /**
+ * Gets the type of the second input
+ *
+ * @return The type of the second input
+ */
+ public TypeInformation<IN2> getInputType2() {
+ return dataStream2.getType();
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 according to keyPosition1 and keyPosition2. Used for
+ * applying function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param keyPosition1
+ * The field used to compute the hashcode of the elements in the
+ * first input stream.
+ * @param keyPosition2
+ * The field used to compute the hashcode of the elements in the
+ * second input stream.
+ * @return @return The transformed {@link ConnectedDataStream}
+ */
+ public ConnectedDataStream<IN1, IN2> groupBy(int keyPosition1, int keyPosition2) {
+ return new ConnectedDataStream<IN1, IN2>(dataStream1.groupBy(keyPosition1),
+ dataStream2.groupBy(keyPosition2));
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 according to keyPositions1 and keyPositions2. Used for
+ * applying function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param keyPositions1
+ * The fields used to group the first input stream.
+ * @param keyPositions2
+ * The fields used to group the second input stream.
+ * @return @return The transformed {@link ConnectedDataStream}
+ */
+ public ConnectedDataStream<IN1, IN2> groupBy(int[] keyPositions1, int[] keyPositions2) {
+ return new ConnectedDataStream<IN1, IN2>(dataStream1.groupBy(keyPositions1),
+ dataStream2.groupBy(keyPositions2));
+ }
+
+ /**
+ * GroupBy operation for connected data stream using key expressions. Groups
+ * the elements of input1 and input2 according to field1 and field2. A field
+ * expression is either the name of a public field or a getter method with
+ * parentheses of the {@link DataStream}S underlying type. A dot can be used
+ * to drill down into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field1
+ * The grouping expression for the first input
+ * @param field2
+ * The grouping expression for the second input
+ * @return The grouped {@link ConnectedDataStream}
+ */
+ public ConnectedDataStream<IN1, IN2> groupBy(String field1, String field2) {
+ return new ConnectedDataStream<IN1, IN2>(dataStream1.groupBy(field1),
+ dataStream2.groupBy(field2));
+ }
+
+ /**
+ * GroupBy operation for connected data stream using key expressions. Groups
+ * the elements of input1 and input2 according to fields1 and fields2. A
+ * field expression is either the name of a public field or a getter method
+ * with parentheses of the {@link DataStream}S underlying type. A dot can be
+ * used to drill down into objects, as in {@code "field1.getInnerField2()" }
+ * .
+ *
+ * @param fields1
+ * The grouping expressions for the first input
+ * @param fields2
+ * The grouping expressions for the second input
+ * @return The grouped {@link ConnectedDataStream}
+ */
+ public ConnectedDataStream<IN1, IN2> groupBy(String[] fields1, String[] fields2) {
+ return new ConnectedDataStream<IN1, IN2>(dataStream1.groupBy(fields1),
+ dataStream2.groupBy(fields2));
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 using keySelector1 and keySelector2. Used for applying
+ * function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param keySelector1
+ * The {@link KeySelector} used for grouping the first input
+ * @param keySelector2
+ * The {@link KeySelector} used for grouping the second input
+ * @return @return The transformed {@link ConnectedDataStream}
+ */
+ public ConnectedDataStream<IN1, IN2> groupBy(KeySelector<IN1, ?> keySelector1,
+ KeySelector<IN2, ?> keySelector2) {
+ return new ConnectedDataStream<IN1, IN2>(dataStream1.groupBy(keySelector1),
+ dataStream2.groupBy(keySelector2));
+ }
+
+ /**
+ * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
+ * the output to a common type. The transformation calls a
+ * {@link CoMapFunction#map1} for each element of the first input and
+ * {@link CoMapFunction#map2} for each element of the second input. Each
+ * CoMapFunction call returns exactly one element. The user can also extend
+ * {@link RichCoMapFunction} to gain access to other features provided by
+ * the {@link RichFuntion} interface.
+ *
+ * @param coMapper
+ * The CoMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed {@link DataStream}
+ */
+ public <OUT> SingleOutputStreamOperator<OUT, ?> map(CoMapFunction<IN1, IN2, OUT> coMapper) {
+ TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoMapFunction.class,
+ coMapper.getClass(), 2, null, null);
+
+ return addCoFunction("Co-Map", outTypeInfo, new CoMapInvokable<IN1, IN2, OUT>(
+ clean(coMapper)));
+
+ }
+
+ /**
+ * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
+ * maps the output to a common type. The transformation calls a
+ * {@link CoFlatMapFunction#flatMap1} for each element of the first input
+ * and {@link CoFlatMapFunction#flatMap2} for each element of the second
+ * input. Each CoFlatMapFunction call returns any number of elements
+ * including none. The user can also extend {@link RichFlatMapFunction} to
+ * gain access to other features provided by the {@link RichFuntion}
+ * interface.
+ *
+ * @param coFlatMapper
+ * The CoFlatMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed {@link DataStream}
+ */
+ public <OUT> SingleOutputStreamOperator<OUT, ?> flatMap(
+ CoFlatMapFunction<IN1, IN2, OUT> coFlatMapper) {
+ TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoFlatMapFunction.class,
+ coFlatMapper.getClass(), 2, null, null);
+
+ return addCoFunction("Co-Flat Map", outTypeInfo, new CoFlatMapInvokable<IN1, IN2, OUT>(
+ clean(coFlatMapper)));
+ }
+
+ /**
+ * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
+ * the outputs to a common type. If the {@link ConnectedDataStream} is
+ * batched or windowed then the reduce transformation is applied on every
+ * sliding batch/window of the data stream. If the connected data stream is
+ * grouped then the reducer is applied on every group of elements sharing
+ * the same key. This type of reduce is much faster than reduceGroup since
+ * the reduce function can be applied incrementally. The user can also
+ * extend the {@link RichCoReduceFunction} to gain access to other features
+ * provided by the {@link RichFuntion} interface.
+ *
+ * @param coReducer
+ * The {@link CoReduceFunction} that will be called for every
+ * element of the inputs.
+ * @return The transformed {@link DataStream}.
+ */
+ public <OUT> SingleOutputStreamOperator<OUT, ?> reduce(CoReduceFunction<IN1, IN2, OUT> coReducer) {
+
+ TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoReduceFunction.class,
+ coReducer.getClass(), 2, null, null);
+
+ return addCoFunction("Co-Reduce", outTypeInfo, getReduceInvokable(clean(coReducer)));
+
+ }
+
+ /**
+ * Applies a CoWindow transformation on the connected DataStreams. The
+ * transformation calls the {@link CoWindowFunction#coWindow} method for for
+ * time aligned windows of the two data streams. System time is used as
+ * default to compute windows.
+ *
+ * @param coWindowFunction
+ * The {@link CoWindowFunction} that will be applied for the time
+ * windows.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ *
+ * @return The transformed {@link DataStream}.
+ */
+ @SuppressWarnings("unchecked")
+ public <OUT> SingleOutputStreamOperator<OUT, ?> windowReduce(
+ CoWindowFunction<IN1, IN2, OUT> coWindowFunction, long windowSize, long slideInterval) {
+ return windowReduce(coWindowFunction, windowSize, slideInterval,
+ (TimestampWrapper<IN1>) SystemTimestamp.getWrapper(),
+ (TimestampWrapper<IN2>) SystemTimestamp.getWrapper());
+ }
+
+ /**
+ * Applies a CoWindow transformation on the connected DataStreams. The
+ * transformation calls the {@link CoWindowFunction#coWindow} method for
+ * time aligned windows of the two data streams. The user can implement
+ * their own time stamps or use the system time by default.
+ *
+ * @param coWindowFunction
+ * The {@link CoWindowFunction} that will be applied for the time
+ * windows.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams. If
+ * system time is used it is milliseconds. User defined time
+ * stamps are assumed to be monotonically increasing.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ *
+ * @param timestamp1
+ * User defined time stamps for the first input.
+ * @param timestamp2
+ * User defined time stamps for the second input.
+ * @return The transformed {@link DataStream}.
+ */
+ public <OUT> SingleOutputStreamOperator<OUT, ?> windowReduce(
+ CoWindowFunction<IN1, IN2, OUT> coWindowFunction, long windowSize, long slideInterval,
+ TimestampWrapper<IN1> timestamp1, TimestampWrapper<IN2> timestamp2) {
+
+ if (windowSize < 1) {
+ throw new IllegalArgumentException("Window size must be positive");
+ }
+ if (slideInterval < 1) {
+ throw new IllegalArgumentException("Slide interval must be positive");
+ }
+
+ TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoWindowFunction.class,
+ coWindowFunction.getClass(), 2, null, null);
+
+ return addCoFunction("Co-Window", outTypeInfo, new CoWindowInvokable<IN1, IN2, OUT>(
+ clean(coWindowFunction), windowSize, slideInterval, timestamp1, timestamp2));
+
+ }
+
+ protected <OUT> CoInvokable<IN1, IN2, OUT> getReduceInvokable(
+ CoReduceFunction<IN1, IN2, OUT> coReducer) {
+ CoReduceInvokable<IN1, IN2, OUT> invokable;
+ if (isGrouped) {
+ invokable = new CoGroupedReduceInvokable<IN1, IN2, OUT>(clean(coReducer), keySelector1,
+ keySelector2);
+ } else {
+ invokable = new CoReduceInvokable<IN1, IN2, OUT>(clean(coReducer));
+ }
+ return invokable;
+ }
+
+ public <OUT> SingleOutputStreamOperator<OUT, ?> addGeneralWindowCombine(
+ CoWindowFunction<IN1, IN2, OUT> coWindowFunction, TypeInformation<OUT> outTypeInfo,
+ long windowSize, long slideInterval, TimestampWrapper<IN1> timestamp1,
+ TimestampWrapper<IN2> timestamp2) {
+
+ if (windowSize < 1) {
+ throw new IllegalArgumentException("Window size must be positive");
+ }
+ if (slideInterval < 1) {
+ throw new IllegalArgumentException("Slide interval must be positive");
+ }
+
+ return addCoFunction("Co-Window", outTypeInfo, new CoWindowInvokable<IN1, IN2, OUT>(
+ clean(coWindowFunction), windowSize, slideInterval, timestamp1, timestamp2));
+
+ }
+
+ public <OUT> SingleOutputStreamOperator<OUT, ?> addCoFunction(String functionName,
+ TypeInformation<OUT> outTypeInfo, CoInvokable<IN1, IN2, OUT> functionInvokable) {
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ SingleOutputStreamOperator<OUT, ?> returnStream = new SingleOutputStreamOperator(
+ environment, functionName, outTypeInfo, functionInvokable);
+
+ dataStream1.streamGraph.addCoTask(returnStream.getId(), functionInvokable,
+ getInputType1(), getInputType2(), outTypeInfo, functionName,
+ environment.getDegreeOfParallelism());
+
+ dataStream1.connectGraph(dataStream1, returnStream.getId(), 1);
+ dataStream1.connectGraph(dataStream2, returnStream.getId(), 2);
+
+ return returnStream;
+ }
+
+ protected ConnectedDataStream<IN1, IN2> copy() {
+ return new ConnectedDataStream<IN1, IN2>(this);
+ }
+
+}
[41/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
deleted file mode 100644
index aed4496..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ /dev/null
@@ -1,1271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.Validate;
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.functions.RichFilterFunction;
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.common.functions.RichReduceFunction;
-import org.apache.flink.api.common.io.OutputFormat;
-import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
-import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.ClosureCleaner;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.io.CsvOutputFormat;
-import org.apache.flink.api.java.io.TextOutputFormat;
-import org.apache.flink.api.java.operators.Keys;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.core.fs.FileSystem.WriteMode;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.streaming.api.StreamGraph;
-import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.datastream.temporaloperator.StreamCrossOperator;
-import org.apache.flink.streaming.api.datastream.temporaloperator.StreamJoinOperator;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
-import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
-import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
-import org.apache.flink.streaming.api.function.sink.FileSinkFunctionByMillis;
-import org.apache.flink.streaming.api.function.sink.PrintSinkFunction;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.api.invokable.SinkInvokable;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.operator.CounterInvokable;
-import org.apache.flink.streaming.api.invokable.operator.FilterInvokable;
-import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable;
-import org.apache.flink.streaming.api.invokable.operator.MapInvokable;
-import org.apache.flink.streaming.api.invokable.operator.StreamReduceInvokable;
-import org.apache.flink.streaming.api.windowing.helper.Count;
-import org.apache.flink.streaming.api.windowing.helper.Delta;
-import org.apache.flink.streaming.api.windowing.helper.Time;
-import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.apache.flink.streaming.partitioner.BroadcastPartitioner;
-import org.apache.flink.streaming.partitioner.DistributePartitioner;
-import org.apache.flink.streaming.partitioner.FieldsPartitioner;
-import org.apache.flink.streaming.partitioner.GlobalPartitioner;
-import org.apache.flink.streaming.partitioner.ShufflePartitioner;
-import org.apache.flink.streaming.partitioner.StreamPartitioner;
-import org.apache.flink.streaming.util.keys.KeySelectorUtil;
-
-/**
- * A DataStream represents a stream of elements of the same type. A DataStream
- * can be transformed into another DataStream by applying a transformation as
- * for example
- * <ul>
- * <li>{@link DataStream#map},</li>
- * <li>{@link DataStream#filter}, or</li>
- * <li>{@link DataStream#aggregate}.</li>
- * </ul>
- *
- * @param <OUT>
- * The type of the DataStream, i.e., the type of the elements of the
- * DataStream.
- */
-public class DataStream<OUT> {
-
- protected static Integer counter = 0;
- protected final StreamExecutionEnvironment environment;
- protected final String id;
- protected final String type;
- protected int degreeOfParallelism;
- protected List<String> userDefinedNames;
- protected StreamPartitioner<OUT> partitioner;
- @SuppressWarnings("rawtypes")
- protected TypeInformation typeInfo;
- protected List<DataStream<OUT>> mergedStreams;
-
- protected final StreamGraph streamGraph;
-
- /**
- * Create a new {@link DataStream} in the given execution environment with
- * partitioning set to forward by default.
- *
- * @param environment
- * StreamExecutionEnvironment
- * @param operatorType
- * The type of the operator in the component
- * @param typeInfo
- * Type of the datastream
- */
- public DataStream(StreamExecutionEnvironment environment, String operatorType,
- TypeInformation<OUT> typeInfo) {
- if (environment == null) {
- throw new NullPointerException("context is null");
- }
-
- counter++;
- this.id = counter.toString();
- this.type = operatorType;
- this.environment = environment;
- this.degreeOfParallelism = environment.getDegreeOfParallelism();
- this.streamGraph = environment.getStreamGraph();
- this.userDefinedNames = new ArrayList<String>();
- this.partitioner = new DistributePartitioner<OUT>(true);
- this.typeInfo = typeInfo;
- this.mergedStreams = new ArrayList<DataStream<OUT>>();
- this.mergedStreams.add(this);
- }
-
- /**
- * Create a new DataStream by creating a copy of another DataStream
- *
- * @param dataStream
- * The DataStream that will be copied.
- */
- public DataStream(DataStream<OUT> dataStream) {
- this.environment = dataStream.environment;
- this.id = dataStream.id;
- this.type = dataStream.type;
- this.degreeOfParallelism = dataStream.degreeOfParallelism;
- this.userDefinedNames = new ArrayList<String>(dataStream.userDefinedNames);
- this.partitioner = dataStream.partitioner;
- this.streamGraph = dataStream.streamGraph;
- this.typeInfo = dataStream.typeInfo;
- this.mergedStreams = new ArrayList<DataStream<OUT>>();
- this.mergedStreams.add(this);
- if (dataStream.mergedStreams.size() > 1) {
- for (int i = 1; i < dataStream.mergedStreams.size(); i++) {
- this.mergedStreams.add(new DataStream<OUT>(dataStream.mergedStreams.get(i)));
- }
- }
-
- }
-
- /**
- * Returns the ID of the {@link DataStream}.
- *
- * @return ID of the DataStream
- */
- public String getId() {
- return id;
- }
-
- /**
- * Gets the degree of parallelism for this operator.
- *
- * @return The parallelism set for this operator.
- */
- public int getParallelism() {
- return this.degreeOfParallelism;
- }
-
- /**
- * Gets the type of the stream.
- *
- * @return The type of the datastream.
- */
- @SuppressWarnings("unchecked")
- public TypeInformation<OUT> getType() {
- return this.typeInfo;
- }
-
- @SuppressWarnings("unchecked")
- public <R> DataStream<R> setType(TypeInformation<R> outType) {
- streamGraph.setOutType(id, outType);
- typeInfo = outType;
- return (DataStream<R>) this;
- }
-
- public <F> F clean(F f) {
- if (getExecutionEnvironment().getConfig().isClosureCleanerEnabled()) {
- ClosureCleaner.clean(f, true);
- }
- ClosureCleaner.ensureSerializable(f);
- return f;
- }
-
- public StreamExecutionEnvironment getExecutionEnvironment() {
- return environment;
- }
-
- /**
- * Creates a new {@link DataStream} by merging {@link DataStream} outputs of
- * the same type with each other. The DataStreams merged using this operator
- * will be transformed simultaneously.
- *
- * @param streams
- * The DataStreams to merge output with.
- * @return The {@link DataStream}.
- */
- public DataStream<OUT> merge(DataStream<OUT>... streams) {
- DataStream<OUT> returnStream = this.copy();
-
- for (DataStream<OUT> stream : streams) {
- for (DataStream<OUT> ds : stream.mergedStreams) {
- validateMerge(ds.getId());
- returnStream.mergedStreams.add(ds.copy());
- }
- }
- return returnStream;
- }
-
- /**
- * Operator used for directing tuples to specific named outputs using an
- * {@link org.apache.flink.streaming.api.collector.OutputSelector}. Calling
- * this method on an operator creates a new {@link SplitDataStream}.
- *
- * @param outputSelector
- * The user defined
- * {@link org.apache.flink.streaming.api.collector.OutputSelector}
- * for directing the tuples.
- * @return The {@link SplitDataStream}
- */
- public SplitDataStream<OUT> split(OutputSelector<OUT> outputSelector) {
- for (DataStream<OUT> ds : this.mergedStreams) {
- streamGraph.setOutputSelector(ds.getId(), clean(outputSelector));
- }
-
- return new SplitDataStream<OUT>(this);
- }
-
- /**
- * Creates a new {@link ConnectedDataStream} by connecting
- * {@link DataStream} outputs of different type with each other. The
- * DataStreams connected using this operators can be used with CoFunctions.
- *
- * @param dataStream
- * The DataStream with which this stream will be joined.
- * @return The {@link ConnectedDataStream}.
- */
- public <R> ConnectedDataStream<OUT, R> connect(DataStream<R> dataStream) {
- return new ConnectedDataStream<OUT, R>(this, dataStream);
- }
-
- /**
- * Groups the elements of a {@link DataStream} by the given key positions to
- * be used with grouped operators like
- * {@link GroupedDataStream#reduce(ReduceFunction)}</p> This operator also
- * affects the partitioning of the stream, by forcing values with the same
- * key to go to the same processing instance.
- *
- * @param fields
- * The position of the fields on which the {@link DataStream}
- * will be grouped.
- * @return The grouped {@link DataStream}
- */
- public GroupedDataStream<OUT> groupBy(int... fields) {
- if (getType() instanceof BasicArrayTypeInfo || getType() instanceof PrimitiveArrayTypeInfo) {
- return groupBy(new KeySelectorUtil.ArrayKeySelector<OUT>(fields));
- } else {
- return groupBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
- }
- }
-
- /**
- * Groups a {@link DataStream} using field expressions. A field expression
- * is either the name of a public field or a getter method with parentheses
- * of the {@link DataStream}S underlying type. A dot can be used to drill
- * down into objects, as in {@code "field1.getInnerField2()" }. This method
- * returns an {@link GroupedDataStream}.</p> This operator also affects the
- * partitioning of the stream, by forcing values with the same key to go to
- * the same processing instance.
- *
- * @param fields
- * One or more field expressions on which the DataStream will be
- * grouped.
- * @return The grouped {@link DataStream}
- **/
- public GroupedDataStream<OUT> groupBy(String... fields) {
-
- return groupBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
-
- }
-
- /**
- * Groups the elements of a {@link DataStream} by the key extracted by the
- * {@link KeySelector} to be used with grouped operators like
- * {@link GroupedDataStream#reduce(ReduceFunction)}.
- * <p/>
- * This operator also affects the partitioning of the stream, by forcing
- * values with the same key to go to the same processing instance.
- *
- * @param keySelector
- * The {@link KeySelector} that will be used to extract keys for
- * the values
- * @return The grouped {@link DataStream}
- */
- public GroupedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
- return new GroupedDataStream<OUT>(this, clean(keySelector));
- }
-
- private GroupedDataStream<OUT> groupBy(Keys<OUT> keys) {
- return new GroupedDataStream<OUT>(this, clean(KeySelectorUtil.getSelectorForKeys(keys,
- getType())));
- }
-
- /**
- * Sets the partitioning of the {@link DataStream} so that the output is
- * partitioned using the given {@link KeySelector}. This setting only
- * effects the how the outputs will be distributed between the parallel
- * instances of the next processing operator.
- *
- * @param keySelector
- * @return
- */
- protected DataStream<OUT> partitionBy(KeySelector<OUT, ?> keySelector) {
- return setConnectionType(new FieldsPartitioner<OUT>(clean(keySelector)));
- }
-
- /**
- * Sets the partitioning of the {@link DataStream} so that the output tuples
- * are broadcasted to every parallel instance of the next component. This
- * setting only effects the how the outputs will be distributed between the
- * parallel instances of the next processing operator.
- *
- * @return The DataStream with broadcast partitioning set.
- */
- public DataStream<OUT> broadcast() {
- return setConnectionType(new BroadcastPartitioner<OUT>());
- }
-
- /**
- * Sets the partitioning of the {@link DataStream} so that the output tuples
- * are shuffled to the next component. This setting only effects the how the
- * outputs will be distributed between the parallel instances of the next
- * processing operator.
- *
- * @return The DataStream with shuffle partitioning set.
- */
- public DataStream<OUT> shuffle() {
- return setConnectionType(new ShufflePartitioner<OUT>());
- }
-
- /**
- * Sets the partitioning of the {@link DataStream} so that the output tuples
- * are forwarded to the local subtask of the next component (whenever
- * possible). This is the default partitioner setting. This setting only
- * effects the how the outputs will be distributed between the parallel
- * instances of the next processing operator.
- *
- * @return The DataStream with shuffle partitioning set.
- */
- public DataStream<OUT> forward() {
- return setConnectionType(new DistributePartitioner<OUT>(true));
- }
-
- /**
- * Sets the partitioning of the {@link DataStream} so that the output tuples
- * are distributed evenly to the next component.This setting only effects
- * the how the outputs will be distributed between the parallel instances of
- * the next processing operator.
- *
- * @return The DataStream with shuffle partitioning set.
- */
- public DataStream<OUT> distribute() {
- return setConnectionType(new DistributePartitioner<OUT>(false));
- }
-
- /**
- * Sets the partitioning of the {@link DataStream} so that the output values
- * all go to the first instance of the next processing operator. Use this
- * setting with care since it might cause a serious performance bottleneck
- * in the application.
- *
- * @return The DataStream with shuffle partitioning set.
- */
- public DataStream<OUT> global() {
- return setConnectionType(new GlobalPartitioner<OUT>());
- }
-
- /**
- * Initiates an iterative part of the program that feeds back data streams.
- * The iterative part needs to be closed by calling
- * {@link IterativeDataStream#closeWith(DataStream)}. The transformation of
- * this IterativeDataStream will be the iteration head. The data stream
- * given to the {@link IterativeDataStream#closeWith(DataStream)} method is
- * the data stream that will be fed back and used as the input for the
- * iteration head. A common usage pattern for streaming iterations is to use
- * output splitting to send a part of the closing data stream to the head.
- * Refer to {@link #split(OutputSelector)} for more information.
- * <p>
- * The iteration edge will be partitioned the same way as the first input of
- * the iteration head.
- * <p>
- * By default a DataStream with iteration will never terminate, but the user
- * can use the maxWaitTime parameter to set a max waiting time for the
- * iteration head. If no data received in the set time, the stream
- * terminates.
- *
- * @return The iterative data stream created.
- */
- public IterativeDataStream<OUT> iterate() {
- return new IterativeDataStream<OUT>(this, 0);
- }
-
- /**
- * Initiates an iterative part of the program that feeds back data streams.
- * The iterative part needs to be closed by calling
- * {@link IterativeDataStream#closeWith(DataStream)}. The transformation of
- * this IterativeDataStream will be the iteration head. The data stream
- * given to the {@link IterativeDataStream#closeWith(DataStream)} method is
- * the data stream that will be fed back and used as the input for the
- * iteration head. A common usage pattern for streaming iterations is to use
- * output splitting to send a part of the closing data stream to the head.
- * Refer to {@link #split(OutputSelector)} for more information.
- * <p>
- * The iteration edge will be partitioned the same way as the first input of
- * the iteration head.
- * <p>
- * By default a DataStream with iteration will never terminate, but the user
- * can use the maxWaitTime parameter to set a max waiting time for the
- * iteration head. If no data received in the set time, the stream
- * terminates.
- *
- * @param maxWaitTimeMillis
- * Number of milliseconds to wait between inputs before shutting
- * down
- *
- * @return The iterative data stream created.
- */
- public IterativeDataStream<OUT> iterate(long maxWaitTimeMillis) {
- return new IterativeDataStream<OUT>(this, maxWaitTimeMillis);
- }
-
- /**
- * Applies a Map transformation on a {@link DataStream}. The transformation
- * calls a {@link MapFunction} for each element of the DataStream. Each
- * MapFunction call returns exactly one element. The user can also extend
- * {@link RichMapFunction} to gain access to other features provided by the
- * {@link org.apache.flink.api.common.functions.RichFunction} interface.
- *
- * @param mapper
- * The MapFunction that is called for each element of the
- * DataStream.
- * @param <R>
- * output type
- * @return The transformed {@link DataStream}.
- */
- public <R> SingleOutputStreamOperator<R, ?> map(MapFunction<OUT, R> mapper) {
-
- TypeInformation<R> outType = TypeExtractor.getMapReturnTypes(clean(mapper), getType());
-
- return transform("Map", outType, new MapInvokable<OUT, R>(clean(mapper)));
- }
-
- /**
- * Applies a FlatMap transformation on a {@link DataStream}. The
- * transformation calls a {@link FlatMapFunction} for each element of the
- * DataStream. Each FlatMapFunction call can return any number of elements
- * including none. The user can also extend {@link RichFlatMapFunction} to
- * gain access to other features provided by the
- * {@link org.apache.flink.api.common.functions.RichFunction} interface.
- *
- * @param flatMapper
- * The FlatMapFunction that is called for each element of the
- * DataStream
- *
- * @param <R>
- * output type
- * @return The transformed {@link DataStream}.
- */
- public <R> SingleOutputStreamOperator<R, ?> flatMap(FlatMapFunction<OUT, R> flatMapper) {
-
- TypeInformation<R> outType = TypeExtractor.getFlatMapReturnTypes(clean(flatMapper),
- getType());
-
- return transform("Flat Map", outType, new FlatMapInvokable<OUT, R>(clean(flatMapper)));
-
- }
-
- /**
- * Applies a reduce transformation on the data stream. The user can also
- * extend the {@link RichReduceFunction} to gain access to other features
- * provided by the
- * {@link org.apache.flink.api.common.functions.RichFunction} interface.
- *
- * @param reducer
- * The {@link ReduceFunction} that will be called for every
- * element of the input values.
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reducer) {
-
- return transform("Reduce", getType(), new StreamReduceInvokable<OUT>(clean(reducer)));
-
- }
-
- /**
- * Applies a Filter transformation on a {@link DataStream}. The
- * transformation calls a {@link FilterFunction} for each element of the
- * DataStream and retains only those element for which the function returns
- * true. Elements for which the function returns false are filtered. The
- * user can also extend {@link RichFilterFunction} to gain access to other
- * features provided by the
- * {@link org.apache.flink.api.common.functions.RichFunction} interface.
- *
- * @param filter
- * The FilterFunction that is called for each element of the
- * DataSet.
- * @return The filtered DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> filter(FilterFunction<OUT> filter) {
- return transform("Filter", getType(), new FilterInvokable<OUT>(clean(filter)));
-
- }
-
- /**
- * Initiates a Project transformation on a {@link Tuple} {@link DataStream}.<br/>
- * <b>Note: Only Tuple DataStreams can be projected.</b></br> The
- * transformation projects each Tuple of the DataSet onto a (sub)set of
- * fields.</br> This method returns a {@link StreamProjection} on which
- * {@link StreamProjection#types(Class)} needs to be called to completed the
- * transformation.
- *
- * @param fieldIndexes
- * The field indexes of the input tuples that are retained. The
- * order of fields in the output tuple corresponds to the order
- * of field indexes.
- * @return A StreamProjection that needs to be converted into a DataStream
- * to complete the project transformation by calling
- * {@link StreamProjection#types(Class)}.
- *
- * @see Tuple
- * @see DataStream
- */
- public StreamProjection<OUT> project(int... fieldIndexes) {
- return new StreamProjection<OUT>(this.copy(), fieldIndexes);
- }
-
- /**
- * Initiates a temporal Cross transformation.<br/>
- * A Cross transformation combines the elements of two {@link DataStream}s
- * into one DataStream over a specified time window. It builds all pair
- * combinations of elements of both DataStreams, i.e., it builds a Cartesian
- * product.
- *
- * <p>
- * This method returns a {@link StreamCrossOperator} on which the
- * {@link StreamCrossOperator#onWindow} should be called to define the
- * window.
- * <p>
- * Call {@link StreamCrossOperator.CrossWindow#with(crossFunction)} to
- * define a custom cross function.
- *
- * @param dataStreamToCross
- * The other DataStream with which this DataStream is crossed.
- * @return A {@link StreamCrossOperator} to continue the definition of the
- * cross transformation.
- *
- */
- public <IN2> StreamCrossOperator<OUT, IN2> cross(DataStream<IN2> dataStreamToCross) {
- return new StreamCrossOperator<OUT, IN2>(this, dataStreamToCross);
- }
-
- /**
- * Initiates a temporal Join transformation. <br/>
- * A temporal Join transformation joins the elements of two
- * {@link DataStream}s on key equality over a specified time window.</br>
- *
- * This method returns a {@link StreamJoinOperator} on which the
- * {@link StreamJoinOperator#onWindow} should be called to define the
- * window, and then the {@link StreamJoinOperator.JoinWindow#where} and
- * {@link StreamJoinOperator.JoinPredicate#equalTo} can be used to define
- * the join keys.</p> The user can also use the
- * {@link StreamJoinOperator.JoinedStream#with(joinFunction)} to apply
- * custom join function.
- *
- * @param other
- * The other DataStream with which this DataStream is joined.
- * @return A {@link StreamJoinOperator} to continue the definition of the
- * Join transformation.
- *
- */
- public <IN2> StreamJoinOperator<OUT, IN2> join(DataStream<IN2> dataStreamToJoin) {
- return new StreamJoinOperator<OUT, IN2>(this, dataStreamToJoin);
- }
-
- /**
- * Applies an aggregation that sums the data stream at the given position.
- *
- * @param positionToSum
- * The position in the data point to sum
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> sum(int positionToSum) {
- checkFieldRange(positionToSum);
- return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(positionToSum,
- getClassAtPos(positionToSum), getType()));
- }
-
- /**
- * Applies an aggregation that that gives the current sum of the pojo data
- * stream at the given field expression. A field expression is either the
- * name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> sum(String field) {
- return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field, getType()));
- }
-
- /**
- * Applies an aggregation that that gives the current minimum of the data
- * stream at the given position.
- *
- * @param positionToMin
- * The position in the data point to minimize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> min(int positionToMin) {
- checkFieldRange(positionToMin);
- return aggregate(ComparableAggregator.getAggregator(positionToMin, getType(),
- AggregationType.MIN));
- }
-
- /**
- * Applies an aggregation that that gives the current minimum of the pojo
- * data stream at the given field expression. A field expression is either
- * the name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> min(String field) {
- return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MIN,
- false));
- }
-
- /**
- * Applies an aggregation that gives the current maximum of the data stream
- * at the given position.
- *
- * @param positionToMax
- * The position in the data point to maximize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> max(int positionToMax) {
- checkFieldRange(positionToMax);
- return aggregate(ComparableAggregator.getAggregator(positionToMax, getType(),
- AggregationType.MAX));
- }
-
- /**
- * Applies an aggregation that that gives the current maximum of the pojo
- * data stream at the given field expression. A field expression is either
- * the name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> max(String field) {
- return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MAX,
- false));
- }
-
- /**
- * Applies an aggregation that that gives the current minimum element of the
- * pojo data stream by the given field expression. A field expression is
- * either the name of a public field or a getter method with parentheses of
- * the {@link DataStream}S underlying type. A dot can be used to drill down
- * into objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @param first
- * If True then in case of field equality the first object will
- * be returned
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(String field, boolean first) {
- return aggregate(ComparableAggregator.getAggregator(field, getType(),
- AggregationType.MINBY, first));
- }
-
- /**
- * Applies an aggregation that that gives the current maximum element of the
- * pojo data stream by the given field expression. A field expression is
- * either the name of a public field or a getter method with parentheses of
- * the {@link DataStream}S underlying type. A dot can be used to drill down
- * into objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @param first
- * If True then in case of field equality the first object will
- * be returned
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(String field, boolean first) {
- return aggregate(ComparableAggregator.getAggregator(field, getType(),
- AggregationType.MAXBY, first));
- }
-
- /**
- * Applies an aggregation that that gives the current element with the
- * minimum value at the given position, if more elements have the minimum
- * value at the given position, the operator returns the first one by
- * default.
- *
- * @param positionToMinBy
- * The position in the data point to minimize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
- return this.minBy(positionToMinBy, true);
- }
-
- /**
- * Applies an aggregation that that gives the current element with the
- * minimum value at the given position, if more elements have the minimum
- * value at the given position, the operator returns the first one by
- * default.
- *
- * @param positionToMinBy
- * The position in the data point to minimize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(String positionToMinBy) {
- return this.minBy(positionToMinBy, true);
- }
-
- /**
- * Applies an aggregation that that gives the current element with the
- * minimum value at the given position, if more elements have the minimum
- * value at the given position, the operator returns either the first or
- * last one, depending on the parameter set.
- *
- * @param positionToMinBy
- * The position in the data point to minimize
- * @param first
- * If true, then the operator return the first element with the
- * minimal value, otherwise returns the last
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy, boolean first) {
- checkFieldRange(positionToMinBy);
- return aggregate(ComparableAggregator.getAggregator(positionToMinBy, getType(),
- AggregationType.MINBY, first));
- }
-
- /**
- * Applies an aggregation that that gives the current element with the
- * maximum value at the given position, if more elements have the maximum
- * value at the given position, the operator returns the first one by
- * default.
- *
- * @param positionToMaxBy
- * The position in the data point to maximize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
- return this.maxBy(positionToMaxBy, true);
- }
-
- /**
- * Applies an aggregation that that gives the current element with the
- * maximum value at the given position, if more elements have the maximum
- * value at the given position, the operator returns the first one by
- * default.
- *
- * @param positionToMaxBy
- * The position in the data point to maximize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(String positionToMaxBy) {
- return this.maxBy(positionToMaxBy, true);
- }
-
- /**
- * Applies an aggregation that that gives the current element with the
- * maximum value at the given position, if more elements have the maximum
- * value at the given position, the operator returns either the first or
- * last one, depending on the parameter set.
- *
- * @param positionToMaxBy
- * The position in the data point to maximize.
- * @param first
- * If true, then the operator return the first element with the
- * maximum value, otherwise returns the last
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
- checkFieldRange(positionToMaxBy);
- return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getType(),
- AggregationType.MAXBY, first));
- }
-
- /**
- * Creates a new DataStream containing the current number (count) of
- * received records.
- *
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<Long, ?> count() {
- TypeInformation<Long> outTypeInfo = TypeExtractor.getForObject(Long.valueOf(0));
-
- return transform("Count", outTypeInfo, new CounterInvokable<OUT>());
- }
-
- /**
- * Create a {@link WindowedDataStream} that can be used to apply
- * transformation like {@link WindowedDataStream#reduce} or aggregations on
- * preset chunks(windows) of the data stream. To define the windows one or
- * more {@link WindowingHelper} such as {@link Time}, {@link Count} and
- * {@link Delta} can be used.</br></br> When applied to a grouped data
- * stream, the windows (evictions) and slide sizes (triggers) will be
- * computed on a per group basis. </br></br> For more advanced control over
- * the trigger and eviction policies please refer to
- * {@link #window(triggers, evicters)} </br> </br> For example to create a
- * sum every 5 seconds in a tumbling fashion:</br>
- * {@code ds.window(Time.of(5, TimeUnit.SECONDS)).sum(field)} </br></br> To
- * create sliding windows use the
- * {@link WindowedDataStream#every(WindowingHelper...)} </br></br> The same
- * example with 3 second slides:</br>
- *
- * {@code ds.window(Time.of(5, TimeUnit.SECONDS)).every(Time.of(3,
- * TimeUnit.SECONDS)).sum(field)}
- *
- * @param policyHelpers
- * Any {@link WindowingHelper} such as {@link Time},
- * {@link Count} and {@link Delta} to define the window.
- * @return A {@link WindowedDataStream} providing further operations.
- */
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public WindowedDataStream<OUT> window(WindowingHelper... policyHelpers) {
- return new WindowedDataStream<OUT>(this, policyHelpers);
- }
-
- /**
- * Create a {@link WindowedDataStream} using the given {@link TriggerPolicy}
- * s and {@link EvictionPolicy}s. Windowing can be used to apply
- * transformation like {@link WindowedDataStream#reduce} or aggregations on
- * preset chunks(windows) of the data stream.</br></br>For most common
- * use-cases please refer to {@link #window(WindowingHelper...)}
- *
- * @param triggers
- * The list of {@link TriggerPolicy}s that will determine how
- * often the user function is called on the window.
- * @param evicters
- * The list of {@link EvictionPolicy}s that will determine the
- * number of elements in each time window.
- * @return A {@link WindowedDataStream} providing further operations.
- */
- public WindowedDataStream<OUT> window(List<TriggerPolicy<OUT>> triggers,
- List<EvictionPolicy<OUT>> evicters) {
- return new WindowedDataStream<OUT>(this, triggers, evicters);
- }
-
- /**
- * Writes a DataStream to the standard output stream (stdout).<br>
- * For each element of the DataStream the result of
- * {@link Object#toString()} is written.
- *
- * @return The closed DataStream.
- */
- public DataStreamSink<OUT> print() {
- PrintSinkFunction<OUT> printFunction = new PrintSinkFunction<OUT>();
- DataStreamSink<OUT> returnStream = addSink(printFunction);
-
- return returnStream;
- }
-
- /**
- * Writes a DataStream to the standard output stream (stderr).<br>
- * For each element of the DataStream the result of
- * {@link Object#toString()} is written.
- *
- * @return The closed DataStream.
- */
- public DataStreamSink<OUT> printToErr() {
- PrintSinkFunction<OUT> printFunction = new PrintSinkFunction<OUT>(true);
- DataStreamSink<OUT> returnStream = addSink(printFunction);
-
- return returnStream;
- }
-
- /**
- * Writes a DataStream to the file specified by path in text format. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written.
- *
- * @param path
- * the path pointing to the location the text file is written to
- *
- * @return the closed DataStream.
- */
- public DataStreamSink<OUT> writeAsText(String path) {
- return writeToFile(new TextOutputFormat<OUT>(new Path(path)), 0L);
- }
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically, in every millis milliseconds. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written.
- *
- * @param path
- * the path pointing to the location the text file is written to
- * @param millis
- * the file update frequency
- *
- * @return the closed DataStream
- */
- public DataStreamSink<OUT> writeAsText(String path, long millis) {
- TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
- return writeToFile(tof, millis);
- }
-
- /**
- * Writes a DataStream to the file specified by path in text format. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written.
- *
- * @param path
- * the path pointing to the location the text file is written to
- * @param writeMode
- * Control the behavior for existing files. Options are
- * NO_OVERWRITE and OVERWRITE.
- *
- * @return the closed DataStream.
- */
- public DataStreamSink<OUT> writeAsText(String path, WriteMode writeMode) {
- TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
- tof.setWriteMode(writeMode);
- return writeToFile(tof, 0L);
- }
-
- /**
- * Writes a DataStream to the file specified by path in text format. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written.
- *
- * @param path
- * the path pointing to the location the text file is written to
- * @param writeMode
- * Controls the behavior for existing files. Options are
- * NO_OVERWRITE and OVERWRITE.
- * @param millis
- * the file update frequency
- *
- * @return the closed DataStream.
- */
- public DataStreamSink<OUT> writeAsText(String path, WriteMode writeMode, long millis) {
- TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
- tof.setWriteMode(writeMode);
- return writeToFile(tof, millis);
- }
-
- /**
- * Writes a DataStream to the file specified by path in csv format. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written. This method can only be used on data streams of tuples.
- *
- * @param path
- * the path pointing to the location the text file is written to
- *
- * @return the closed DataStream
- */
- @SuppressWarnings("unchecked")
- public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path) {
- Validate.isTrue(getType().isTupleType(),
- "The writeAsCsv() method can only be used on data sets of tuples.");
- CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
- CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
- return writeToFile((OutputFormat<OUT>) of, 0L);
- }
-
- /**
- * Writes a DataStream to the file specified by path in csv format. The
- * writing is performed periodically, in every millis milliseconds. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written. This method can only be used on data streams of tuples.
- *
- * @param path
- * the path pointing to the location the text file is written to
- * @param millis
- * the file update frequency
- *
- * @return the closed DataStream
- */
- @SuppressWarnings("unchecked")
- public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, long millis) {
- Validate.isTrue(getType().isTupleType(),
- "The writeAsCsv() method can only be used on data sets of tuples.");
- CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
- CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
- return writeToFile((OutputFormat<OUT>) of, millis);
- }
-
- /**
- * Writes a DataStream to the file specified by path in csv format. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written. This method can only be used on data streams of tuples.
- *
- * @param path
- * the path pointing to the location the text file is written to
- * @param writeMode
- * Controls the behavior for existing files. Options are
- * NO_OVERWRITE and OVERWRITE.
- *
- * @return the closed DataStream
- */
- @SuppressWarnings("unchecked")
- public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, WriteMode writeMode) {
- Validate.isTrue(getType().isTupleType(),
- "The writeAsCsv() method can only be used on data sets of tuples.");
- CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
- CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
- if (writeMode != null) {
- of.setWriteMode(writeMode);
- }
- return writeToFile((OutputFormat<OUT>) of, 0L);
- }
-
- /**
- * Writes a DataStream to the file specified by path in csv format. The
- * writing is performed periodically, in every millis milliseconds. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written. This method can only be used on data streams of tuples.
- *
- * @param path
- * the path pointing to the location the text file is written to
- * @param writeMode
- * Controls the behavior for existing files. Options are
- * NO_OVERWRITE and OVERWRITE.
- * @param millis
- * the file update frequency
- *
- * @return the closed DataStream
- */
- @SuppressWarnings("unchecked")
- public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, WriteMode writeMode,
- long millis) {
- Validate.isTrue(getType().isTupleType(),
- "The writeAsCsv() method can only be used on data sets of tuples.");
- CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
- CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
- if (writeMode != null) {
- of.setWriteMode(writeMode);
- }
- return writeToFile((OutputFormat<OUT>) of, millis);
- }
-
- private DataStreamSink<OUT> writeToFile(OutputFormat<OUT> format, long millis) {
- DataStreamSink<OUT> returnStream = addSink(new FileSinkFunctionByMillis<OUT>(format, millis));
- return returnStream;
- }
-
- protected SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregate) {
-
- StreamReduceInvokable<OUT> invokable = new StreamReduceInvokable<OUT>(aggregate);
-
- SingleOutputStreamOperator<OUT, ?> returnStream = transform("Aggregation", getType(),
- invokable);
-
- return returnStream;
- }
-
- /**
- * Method for passing user defined invokables along with the type
- * informations that will transform the DataStream.
- *
- * @param operatorName
- * name of the operator, for logging purposes
- * @param outTypeInfo
- * the output type of the operator
- * @param invokable
- * the object containing the transformation logic
- * @param <R>
- * type of the return stream
- * @return the data stream constructed
- */
- public <R> SingleOutputStreamOperator<R, ?> transform(String operatorName,
- TypeInformation<R> outTypeInfo, StreamInvokable<OUT, R> invokable) {
- DataStream<OUT> inputStream = this.copy();
- @SuppressWarnings({ "unchecked", "rawtypes" })
- SingleOutputStreamOperator<R, ?> returnStream = new SingleOutputStreamOperator(environment,
- operatorName, outTypeInfo, invokable);
-
- streamGraph.addStreamVertex(returnStream.getId(), invokable, getType(), outTypeInfo,
- operatorName, returnStream.getParallelism());
-
- connectGraph(inputStream, returnStream.getId(), 0);
-
- return returnStream;
- }
-
- /**
- * Internal function for setting the partitioner for the DataStream
- *
- * @param partitioner
- * Partitioner to set.
- * @return The modified DataStream.
- */
- protected DataStream<OUT> setConnectionType(StreamPartitioner<OUT> partitioner) {
- DataStream<OUT> returnStream = this.copy();
-
- for (DataStream<OUT> stream : returnStream.mergedStreams) {
- stream.partitioner = partitioner;
- }
-
- return returnStream;
- }
-
- /**
- * Internal function for assembling the underlying
- * {@link org.apache.flink.runtime.jobgraph.JobGraph} of the job. Connects
- * the outputs of the given input stream to the specified output stream
- * given by the outputID.
- *
- * @param inputStream
- * input data stream
- * @param outputID
- * ID of the output
- * @param typeNumber
- * Number of the type (used at co-functions)
- */
- protected <X> void connectGraph(DataStream<X> inputStream, String outputID, int typeNumber) {
- for (DataStream<X> stream : inputStream.mergedStreams) {
- streamGraph.setEdge(stream.getId(), outputID, stream.partitioner, typeNumber,
- inputStream.userDefinedNames);
- }
-
- }
-
- /**
- * Adds the given sink to this DataStream. Only streams with sinks added
- * will be executed once the {@link StreamExecutionEnvironment#execute()}
- * method is called.
- *
- * @param sinkFunction
- * The object containing the sink's invoke function.
- * @return The closed DataStream.
- */
- public DataStreamSink<OUT> addSink(SinkFunction<OUT> sinkFunction) {
-
- StreamInvokable<OUT, OUT> sinkInvokable = new SinkInvokable<OUT>(clean(sinkFunction));
-
- DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", getType(),
- sinkInvokable);
-
- streamGraph.addStreamVertex(returnStream.getId(), sinkInvokable, getType(), null,
- "Stream Sink", returnStream.getParallelism());
-
- this.connectGraph(this.copy(), returnStream.getId(), 0);
-
- return returnStream;
- }
-
- /**
- * Gets the class of the field at the given position
- *
- * @param pos
- * Position of the field
- * @return The class of the field
- */
- @SuppressWarnings("rawtypes")
- protected Class<?> getClassAtPos(int pos) {
- Class<?> type;
- TypeInformation<OUT> outTypeInfo = getType();
- if (outTypeInfo.isTupleType()) {
- type = ((TupleTypeInfo) outTypeInfo).getTypeAt(pos).getTypeClass();
-
- } else if (outTypeInfo instanceof BasicArrayTypeInfo) {
-
- type = ((BasicArrayTypeInfo) outTypeInfo).getComponentTypeClass();
-
- } else if (outTypeInfo instanceof PrimitiveArrayTypeInfo) {
- Class<?> clazz = outTypeInfo.getTypeClass();
- if (clazz == boolean[].class) {
- type = Boolean.class;
- } else if (clazz == short[].class) {
- type = Short.class;
- } else if (clazz == int[].class) {
- type = Integer.class;
- } else if (clazz == long[].class) {
- type = Long.class;
- } else if (clazz == float[].class) {
- type = Float.class;
- } else if (clazz == double[].class) {
- type = Double.class;
- } else if (clazz == char[].class) {
- type = Character.class;
- } else {
- throw new IndexOutOfBoundsException("Type could not be determined for array");
- }
-
- } else if (pos == 0) {
- type = outTypeInfo.getTypeClass();
- } else {
- throw new IndexOutOfBoundsException("Position is out of range");
- }
- return type;
- }
-
- /**
- * Checks if the given field position is allowed for the output type
- *
- * @param pos
- * Position to check
- */
- protected void checkFieldRange(int pos) {
- try {
- getClassAtPos(pos);
- } catch (IndexOutOfBoundsException e) {
- throw new RuntimeException("Selected field is out of range");
-
- }
- }
-
- private void validateMerge(String id) {
- for (DataStream<OUT> ds : this.mergedStreams) {
- if (ds.getId().equals(id)) {
- throw new RuntimeException("A DataStream cannot be merged with itself");
- }
- }
- }
-
- /**
- * Creates a copy of the {@link DataStream}
- *
- * @return The copy
- */
- public DataStream<OUT> copy() {
- return new DataStream<OUT>(this);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
deleted file mode 100755
index f064332..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-
-/**
- * Represents the end of a DataStream.
- *
- * @param <IN>
- * The type of the DataStream closed by the sink.
- */
-public class DataStreamSink<IN> extends SingleOutputStreamOperator<IN, DataStreamSink<IN>> {
-
- protected DataStreamSink(StreamExecutionEnvironment environment, String operatorType,
- TypeInformation<IN> outTypeInfo, StreamInvokable<?,?> invokable) {
- super(environment, operatorType, outTypeInfo, invokable);
- }
-
- protected DataStreamSink(DataStream<IN> dataStream) {
- super(dataStream);
- }
-
- @Override
- public DataStreamSink<IN> copy() {
- throw new RuntimeException("Data stream sinks cannot be copied");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
deleted file mode 100755
index b596cbd..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-
-/**
- * The DataStreamSource represents the starting point of a DataStream.
- *
- * @param <OUT>
- * Type of the DataStream created.
- */
-public class DataStreamSource<OUT> extends SingleOutputStreamOperator<OUT, DataStreamSource<OUT>> {
-
- boolean isParallel;
-
- public DataStreamSource(StreamExecutionEnvironment environment, String operatorType,
- TypeInformation<OUT> outTypeInfo, StreamInvokable<?, ?> invokable, boolean isParallel) {
- super(environment, operatorType, outTypeInfo, invokable);
- this.isParallel = isParallel;
- }
-
- @Override
- public DataStreamSource<OUT> setParallelism(int dop) {
- if (dop > 1 && !isParallel) {
- throw new IllegalArgumentException("Source: " + this.id + " is not a parallel source");
- } else {
- return (DataStreamSource<OUT>) super.setParallelism(dop);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
deleted file mode 100755
index c871c20..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.functions.RichReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
-import org.apache.flink.streaming.api.invokable.operator.GroupedReduceInvokable;
-import org.apache.flink.streaming.partitioner.StreamPartitioner;
-
-/**
- * A GroupedDataStream represents a {@link DataStream} which has been
- * partitioned by the given {@link KeySelector}. Operators like {@link #reduce},
- * {@link #batchReduce} etc. can be applied on the {@link GroupedDataStream} to
- * get additional functionality by the grouping.
- *
- * @param <OUT>
- * The output type of the {@link GroupedDataStream}.
- */
-public class GroupedDataStream<OUT> extends DataStream<OUT> {
-
- KeySelector<OUT, ?> keySelector;
-
- protected GroupedDataStream(DataStream<OUT> dataStream, KeySelector<OUT, ?> keySelector) {
- super(dataStream.partitionBy(keySelector));
- this.keySelector = keySelector;
- }
-
- protected GroupedDataStream(GroupedDataStream<OUT> dataStream) {
- super(dataStream);
- this.keySelector = dataStream.keySelector;
- }
-
- public KeySelector<OUT, ?> getKeySelector() {
- return this.keySelector;
- }
-
- /**
- * Applies a reduce transformation on the grouped data stream grouped on by
- * the given key position. The {@link ReduceFunction} will receive input
- * values based on the key value. Only input values with the same key will
- * go to the same reducer.The user can also extend
- * {@link RichReduceFunction} to gain access to other features provided by
- * the {@link RichFuntion} interface.
- *
- * @param reducer
- * The {@link ReduceFunction} that will be called for every
- * element of the input values with the same key.
- * @return The transformed DataStream.
- */
- @Override
- public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reducer) {
- return transform("Grouped Reduce", getType(), new GroupedReduceInvokable<OUT>(clean(reducer),
- keySelector));
- }
-
- /**
- * Applies an aggregation that sums the grouped data stream at the given
- * position, grouped by the given key position. Input values with the same
- * key will be summed.
- *
- * @param positionToSum
- * The position in the data point to sum
- * @return The transformed DataStream.
- */
- @Override
- public SingleOutputStreamOperator<OUT, ?> sum(final int positionToSum) {
- return super.sum(positionToSum);
- }
-
- /**
- * Applies an aggregation that gives the minimum of the grouped data stream
- * at the given position, grouped by the given key position. Input values
- * with the same key will be minimized.
- *
- * @param positionToMin
- * The position in the data point to minimize
- * @return The transformed DataStream.
- */
- @Override
- public SingleOutputStreamOperator<OUT, ?> min(final int positionToMin) {
- return super.min(positionToMin);
- }
-
- /**
- * Applies an aggregation that that gives the current element with the
- * minimum value at the given position for each group on a grouped data
- * stream. If more elements have the minimum value at the given position,
- * the operator returns the first one by default.
- *
- * @param positionToMinBy
- * The position in the data point to minimize
- * @return The transformed DataStream.
- */
- @Override
- public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
- return super.minBy(positionToMinBy);
- }
-
- /**
- * Applies an aggregation that that gives the current element with the
- * minimum value at the given position for each group on a grouped data
- * stream. If more elements have the minimum value at the given position,
- * the operator returns either the first or last one depending on the
- * parameters.
- *
- * @param positionToMinBy
- * The position in the data point to minimize
- * @param first
- * If true, then the operator return the first element with the
- * maximum value, otherwise returns the last
- * @return The transformed DataStream.
- */
- @Override
- public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy, boolean first) {
- return super.minBy(positionToMinBy, first);
- }
-
- /**
- * Applies an aggregation that gives the maximum of the grouped data stream
- * at the given position, grouped by the given key position. Input values
- * with the same key will be maximized.
- *
- * @param positionToMax
- * The position in the data point to maximize
- * @return The transformed DataStream.
- */
- @Override
- public SingleOutputStreamOperator<OUT, ?> max(final int positionToMax) {
- return super.max(positionToMax);
- }
-
- /**
- * Applies an aggregation that that gives the current element with the
- * maximum value at the given position for each group on a grouped data
- * stream. If more elements have the maximum value at the given position,
- * the operator returns the first one by default.
- *
- * @param positionToMaxBy
- * The position in the data point to maximize
- * @return The transformed DataStream.
- */
- @Override
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
- return super.maxBy(positionToMaxBy);
- }
-
- /**
- * Applies an aggregation that that gives the current element with the
- * maximum value at the given position for each group on a grouped data
- * stream. If more elements have the maximum value at the given position,
- * the operator returns either the first or last one depending on the
- * parameters.
- *
- * @param positionToMaxBy
- * The position in the data point to maximize
- * @param first
- * If true, then the operator return the first element with the
- * maximum value, otherwise returns the last
- * @return The transformed DataStream.
- */
- @Override
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
- return super.maxBy(positionToMaxBy, first);
- }
-
- @Override
- protected SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregate) {
-
- GroupedReduceInvokable<OUT> invokable = new GroupedReduceInvokable<OUT>(clean(aggregate),
- keySelector);
-
- SingleOutputStreamOperator<OUT, ?> returnStream = transform("Grouped Aggregation", getType(),
- invokable);
-
- return returnStream;
- }
-
- @Override
- protected DataStream<OUT> setConnectionType(StreamPartitioner<OUT> partitioner) {
- System.out.println("Setting the partitioning after groupBy can affect the grouping");
- return super.setConnectionType(partitioner);
- }
-
- @Override
- public GroupedDataStream<OUT> copy() {
- return new GroupedDataStream<OUT>(this);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
deleted file mode 100644
index 306bfe8..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-
-/**
- * The iterative data stream represents the start of an iteration in a
- * {@link DataStream}.
- *
- * @param <IN>
- * Type of the DataStream
- */
-public class IterativeDataStream<IN> extends
- SingleOutputStreamOperator<IN, IterativeDataStream<IN>> {
-
- static Integer iterationCount = 0;
- protected Integer iterationID;
- protected long waitTime;
-
- protected IterativeDataStream(DataStream<IN> dataStream, long maxWaitTime) {
- super(dataStream);
- setBufferTimeout(dataStream.environment.getBufferTimeout());
- iterationID = iterationCount;
- iterationCount++;
- waitTime = maxWaitTime;
- }
-
- protected IterativeDataStream(DataStream<IN> dataStream, Integer iterationID, long waitTime) {
- super(dataStream);
- this.iterationID = iterationID;
- this.waitTime = waitTime;
- }
-
- /**
- * Closes the iteration. This method defines the end of the iterative
- * program part that will be fed back to the start of the iteration. </br>
- * </br>A common usage pattern for streaming iterations is to use output
- * splitting to send a part of the closing data stream to the head. Refer to
- * {@link SingleOutputStreamOperator#split(OutputSelector)} for more
- * information.
- *
- *
- * @param iterationResult
- * The data stream that is fed back to the next iteration head.
- * @return Returns the stream that was fed back to the iteration. In most
- * cases no further transformation are applied on this stream.
- *
- */
- public DataStream<IN> closeWith(DataStream<IN> iterationTail) {
- DataStream<IN> iterationSink = new DataStreamSink<IN>(environment, "Iteration Sink", null,
- null);
-
- // We add an iteration sink to the tail which will send tuples to the
- // iteration head
- streamGraph.addIterationTail(iterationSink.getId(), iterationTail.getId(), iterationID,
- waitTime);
-
- connectGraph(iterationTail.forward(), iterationSink.getId(), 0);
- return iterationTail;
- }
-
- @Override
- public <R> SingleOutputStreamOperator<R, ?> transform(String operatorName,
- TypeInformation<R> outTypeInfo, StreamInvokable<IN, R> invokable) {
-
- // We call the superclass tranform method
- SingleOutputStreamOperator<R, ?> returnStream = super.transform(operatorName, outTypeInfo,
- invokable);
-
- // Then we add a source that will take care of receiving feedback tuples
- // from the tail
- addIterationSource(returnStream);
-
- return returnStream;
- }
-
- private <X> void addIterationSource(DataStream<X> dataStream) {
-
- DataStream<X> iterationSource = new DataStreamSource<X>(environment, "Iteration Source",
- null, null, true);
-
- streamGraph.addIterationHead(iterationSource.getId(), dataStream.getId(), iterationID,
- dataStream.getParallelism(), waitTime);
- }
-
- @Override
- public IterativeDataStream<IN> copy() {
- return new IterativeDataStream<IN>(this, iterationID, waitTime);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
deleted file mode 100755
index dcfd6fe..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.flink.api.common.functions.RichFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
-import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
-import org.apache.flink.streaming.state.OperatorState;
-
-/**
- * The SingleOutputStreamOperator represents a user defined transformation
- * applied on a {@link DataStream} with one predefined output type.
- *
- * @param <OUT>
- * Output type of the operator.
- * @param <O>
- * Type of the operator.
- */
-public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperator<OUT, O>> extends
- DataStream<OUT> {
-
- protected boolean isSplit;
- protected StreamInvokable<?, ?> invokable;
-
- protected SingleOutputStreamOperator(StreamExecutionEnvironment environment,
- String operatorType, TypeInformation<OUT> outTypeInfo, StreamInvokable<?, ?> invokable) {
- super(environment, operatorType, outTypeInfo);
- setBufferTimeout(environment.getBufferTimeout());
- this.isSplit = false;
- this.invokable = invokable;
- }
-
- @SuppressWarnings("unchecked")
- protected SingleOutputStreamOperator(DataStream<OUT> dataStream) {
- super(dataStream);
- if (dataStream instanceof SingleOutputStreamOperator) {
- this.isSplit = ((SingleOutputStreamOperator<OUT, ?>) dataStream).isSplit;
- this.invokable = ((SingleOutputStreamOperator<OUT, ?>) dataStream).invokable;
- }
- }
-
- @SuppressWarnings("unchecked")
- public <R> SingleOutputStreamOperator<R, ?> setType(TypeInformation<R> outType) {
- streamGraph.setOutType(id, outType);
- typeInfo = outType;
- return (SingleOutputStreamOperator<R, ?>) this;
- }
-
- /**
- * Sets the degree of parallelism for this operator. The degree must be 1 or
- * more.
- *
- * @param dop
- * The degree of parallelism for this operator.
- * @return The operator with set degree of parallelism.
- */
- public SingleOutputStreamOperator<OUT, O> setParallelism(int dop) {
- if (dop < 1) {
- throw new IllegalArgumentException("The parallelism of an operator must be at least 1.");
- }
- this.degreeOfParallelism = dop;
-
- streamGraph.setParallelism(id, degreeOfParallelism);
-
- return this;
- }
-
- /**
- * Sets the maximum time frequency (ms) for the flushing of the output
- * buffer. By default the output buffers flush only when they are full.
- *
- * @param timeoutMillis
- * The maximum time between two output flushes.
- * @return The operator with buffer timeout set.
- */
- public SingleOutputStreamOperator<OUT, O> setBufferTimeout(long timeoutMillis) {
- streamGraph.setBufferTimeout(id, timeoutMillis);
- return this;
- }
-
- /**
- * This is a beta feature </br></br> Register an operator state for this
- * operator by the given name. This name can be used to retrieve the state
- * during runtime using {@link StreamingRuntimeContext#getState(String)}. To
- * obtain the {@link StreamingRuntimeContext} from the user-defined function
- * use the {@link RichFunction#getRuntimeContext()} method.
- *
- * @param name
- * The name of the operator state.
- * @param state
- * The state to be registered for this name.
- * @return The data stream with state registered.
- */
- protected SingleOutputStreamOperator<OUT, O> registerState(String name, OperatorState<?> state) {
- streamGraph.addOperatorState(getId(), name, state);
- return this;
- }
-
- /**
- * This is a beta feature </br></br> Register operator states for this
- * operator provided in a map. The registered states can be retrieved during
- * runtime using {@link StreamingRuntimeContext#getState(String)}. To obtain
- * the {@link StreamingRuntimeContext} from the user-defined function use
- * the {@link RichFunction#getRuntimeContext()} method.
- *
- * @param states
- * The map containing the states that will be registered.
- * @return The data stream with states registered.
- */
- protected SingleOutputStreamOperator<OUT, O> registerState(Map<String, OperatorState<?>> states) {
- for (Entry<String, OperatorState<?>> entry : states.entrySet()) {
- streamGraph.addOperatorState(getId(), entry.getKey(), entry.getValue());
- }
-
- return this;
- }
-
- @SuppressWarnings("unchecked")
- public SingleOutputStreamOperator<OUT, O> broadcast() {
- return (SingleOutputStreamOperator<OUT, O>) super.broadcast();
- }
-
- @SuppressWarnings("unchecked")
- public SingleOutputStreamOperator<OUT, O> shuffle() {
- return (SingleOutputStreamOperator<OUT, O>) super.shuffle();
- }
-
- @SuppressWarnings("unchecked")
- public SingleOutputStreamOperator<OUT, O> forward() {
- return (SingleOutputStreamOperator<OUT, O>) super.forward();
- }
-
- @SuppressWarnings("unchecked")
- public SingleOutputStreamOperator<OUT, O> distribute() {
- return (SingleOutputStreamOperator<OUT, O>) super.distribute();
- }
-
- @SuppressWarnings("unchecked")
- public SingleOutputStreamOperator<OUT, O> global() {
- return (SingleOutputStreamOperator<OUT, O>) super.global();
- }
-
- @Override
- public SingleOutputStreamOperator<OUT, O> copy() {
- return new SingleOutputStreamOperator<OUT, O>(this);
- }
-
- public SingleOutputStreamOperator<OUT, O> setChainingStrategy(ChainingStrategy strategy) {
- this.invokable.setChainingStrategy(strategy);
- return this;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
deleted file mode 100755
index 97458a8..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import java.util.Arrays;
-
-import org.apache.flink.streaming.api.collector.OutputSelector;
-
-/**
- * The SplitDataStream represents an operator that has been split using an
- * {@link OutputSelector}. Named outputs can be selected using the
- * {@link #select} function. To apply transformation on the whole output simply
- * call the transformation on the SplitDataStream
- *
- * @param <OUT>
- * The type of the output.
- */
-public class SplitDataStream<OUT> extends DataStream<OUT> {
-
- protected SplitDataStream(DataStream<OUT> dataStream) {
- super(dataStream);
- }
-
- /**
- * Sets the output names for which the next operator will receive values.
- *
- * @param outputNames
- * The output names for which the operator will receive the
- * input.
- * @return Returns the selected DataStream
- */
- public DataStream<OUT> select(String... outputNames) {
- return selectOutput(outputNames);
- }
-
- private DataStream<OUT> selectOutput(String[] outputNames) {
- for (String outName : outputNames) {
- if (outName == null) {
- throw new RuntimeException("Selected names must not be null");
- }
- }
-
- DataStream<OUT> returnStream = copy();
-
- for (DataStream<OUT> ds : returnStream.mergedStreams) {
- ds.userDefinedNames = Arrays.asList(outputNames);
- }
- return returnStream;
- }
-
-}
[30/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/pom.xml b/flink-addons/flink-streaming/flink-streaming-examples/pom.xml
deleted file mode 100644
index 3661726..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/pom.xml
+++ /dev/null
@@ -1,518 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-parent</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-streaming-examples</artifactId>
- <name>flink-streaming-examples</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-scala</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java-examples</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-connectors</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <!-- get default data from flink-java-examples package -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.9</version><!--$NO-MVN-MAN-VER$-->
- <executions>
- <execution>
- <id>unpack</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>unpack</goal>
- </goals>
- <configuration>
- <artifactItems>
- <artifactItem>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java-examples</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <overWrite>false</overWrite>
- <outputDirectory>${project.build.directory}/classes</outputDirectory>
- <includes>org/apache/flink/examples/java/wordcount/util/WordCountData.class</includes>
- </artifactItem>
- </artifactItems>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- self-contained jars for each example -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
-
- <executions>
-
- <!-- Iteration -->
- <execution>
- <id>Iteration</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>Iteration</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.iteration.IterateExample</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/iteration/*.class</include>
- </includes>
- </configuration>
- </execution>
-
- <!-- IncrementalLearning -->
- <execution>
- <id>IncrementalLearning</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>IncrementalLearning</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.ml.IncrementalLearningSkeleton</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/ml/*.class</include>
- </includes>
- </configuration>
- </execution>
-
- <!-- Twitter -->
- <execution>
- <id>Twitter</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>Twitter</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.twitter.TwitterStream</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/twitter/*.class</include>
- <include>org/apache/flink/streaming/examples/twitter/util/*.class</include>
- </includes>
- </configuration>
- </execution>
-
- <!-- WindowJoin -->
- <execution>
- <id>WindowJoin</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>WindowJoin</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.join.WindowJoin</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/join/*.class</include>
- </includes>
- </configuration>
- </execution>
-
- <!-- WordCountPOJO -->
- <execution>
- <id>WordCountPOJO</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>WordCountPOJO</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.wordcount.PojoExample</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/wordcount/PojoExample.class</include>
- <include>org/apache/flink/streaming/examples/wordcount/PojoExample$*.class</include>
- <include>org/apache/flink/examples/java/wordcount/util/WordCountData.class</include>
- </includes>
- </configuration>
- </execution>
-
- <!-- WordCount -->
- <execution>
- <id>WordCount</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>WordCount</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.wordcount.WordCount</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/wordcount/WordCount.class</include>
- <include>org/apache/flink/streaming/examples/wordcount/WordCount$*.class</include>
- <include>org/apache/flink/examples/java/wordcount/util/WordCountData.class</include>
- </includes>
- </configuration>
- </execution>
-
- <!-- SocketTextStreamWordCount -->
- <execution>
- <id>SocketTextStreamWordCount</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>SocketTextStreamWordCount</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.socket.SocketTextStreamWordCount</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.class</include>
- <include>org/apache/flink/streaming/examples/wordcount/WordCount$Tokenizer.class</include>
- </includes>
- </configuration>
- </execution>
-
- <!-- DeltaExract -->
- <execution>
- <id>DeltaExract</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>DeltaExract</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.windowing.DeltaExtractExample</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/windowing/DeltaExtractExample.class</include>
- <include>org/apache/flink/streaming/examples/windowing/DeltaExtractExample$*.class</include>
- </includes>
- </configuration>
- </execution>
-
- <!-- MultiplePolicies -->
- <execution>
- <id>MultiplePolicies</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>MultiplePolicies</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.windowing.MultiplePoliciesExample</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.class</include>
- <include>org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample$*.class</include>
- </includes>
- </configuration>
- </execution>
-
- <!-- SlidingExample -->
- <execution>
- <id>SlidingExample</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>SlidingExample</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.windowing.SlidingExample</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/windowing/SlidingExample.class</include>
- <include>org/apache/flink/streaming/examples/windowing/SlidingExample$*.class</include>
- </includes>
- </configuration>
- </execution>
-
- <!-- TimeWindowing -->
- <execution>
- <id>TimeWindowing</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>TimeWindowing</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.windowing.TimeWindowingExample</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/windowing/TimeWindowingExample.class</include>
- <include>org/apache/flink/streaming/examples/windowing/TimeWindowingExample$*.class</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
-<!-- Scala Compiler -->
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>3.1.4</version>
- <executions>
- <!-- Run scala compiler in the process-resources phase, so that dependencies on
- scala classes can be resolved later in the (Java) compile phase -->
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
-
- <!-- Run scala compiler in the process-test-resources phase, so that dependencies on
- scala classes can be resolved later in the (Java) test-compile phase -->
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <jvmArgs>
- <jvmArg>-Xms128m</jvmArg>
- <jvmArg>-Xmx512m</jvmArg>
- </jvmArgs>
- <compilerPlugins>
- <compilerPlugin>
- <groupId>org.scalamacros</groupId>
- <artifactId>paradise_${scala.version}</artifactId>
- <version>${scala.macros.version}</version>
- </compilerPlugin>
- </compilerPlugins>
- </configuration>
- </plugin>
-
- <!-- Eclipse Integration -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.8</version>
- <configuration>
- <downloadSources>true</downloadSources>
- <projectnatures>
- <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
- <projectnature>org.eclipse.jdt.core.javanature</projectnature>
- </projectnatures>
- <buildcommands>
- <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
- </buildcommands>
- <classpathContainers>
- <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
- <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
- </classpathContainers>
- <excludes>
- <exclude>org.scala-lang:scala-library</exclude>
- <exclude>org.scala-lang:scala-compiler</exclude>
- </excludes>
- <sourceIncludes>
- <sourceInclude>**/*.scala</sourceInclude>
- <sourceInclude>**/*.java</sourceInclude>
- </sourceIncludes>
- </configuration>
- </plugin>
-
- <!-- Adding scala source directories to build path -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <!-- Add src/main/scala to eclipse build path -->
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/main/scala</source>
- </sources>
- </configuration>
- </execution>
- <!-- Add src/test/scala to eclipse build path -->
- <execution>
- <id>add-test-source</id>
- <phase>generate-test-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/test/scala</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <version>0.5.0</version>
- <executions>
- <execution>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <verbose>false</verbose>
- <failOnViolation>true</failOnViolation>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <failOnWarning>false</failOnWarning>
- <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
- <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
- <configLocation>${project.basedir}/../../../tools/maven/scalastyle-config.xml</configLocation>
- <outputFile>${project.basedir}/scalastyle-output.xml</outputFile>
- <outputEncoding>UTF-8</outputEncoding>
- </configuration>
- </plugin>
-
- </plugins>
-
- <pluginManagement>
- <plugins>
- <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
- <plugin>
- <groupId>org.eclipse.m2e</groupId>
- <artifactId>lifecycle-mapping</artifactId>
- <version>1.0.0</version>
- <configuration>
- <lifecycleMappingMetadata>
- <pluginExecutions>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <versionRange>[2.9,)</versionRange>
- <goals>
- <goal>unpack</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore/>
- </action>
- </pluginExecution>
- </pluginExecutions>
- </lifecycleMappingMetadata>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
-
- </build>
-
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
deleted file mode 100644
index 998e818..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.iteration;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.IterativeDataStream;
-import org.apache.flink.streaming.api.datastream.SplitDataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-
-/**
- * Example illustrating iterations in Flink streaming.
- *
- * <p>
- * The program sums up random numbers and counts additions it performs to reach
- * a specific threshold in an iterative streaming fashion.
- * </p>
- *
- * <p>
- * This example shows how to use:
- * <ul>
- * <li>streaming iterations,
- * <li>buffer timeout to enhance latency,
- * <li>directed outputs.
- * </ul>
- */
-public class IterateExample {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- // set up input for the stream of (0,0) pairs
- List<Tuple2<Double, Integer>> input = new ArrayList<Tuple2<Double, Integer>>();
- for (int i = 0; i < 1000; i++) {
- input.add(new Tuple2<Double, Integer>(0., 0));
- }
-
- // obtain execution environment and set setBufferTimeout(0) to enable
- // continuous flushing of the output buffers (lowest latency)
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment()
- .setBufferTimeout(1);
-
- // create an iterative data stream from the input with 5 second timeout
- IterativeDataStream<Tuple2<Double, Integer>> it = env.fromCollection(input).shuffle()
- .iterate(5000);
-
- // apply the step function to add new random value to the tuple and to
- // increment the counter and split the output with the output selector
- SplitDataStream<Tuple2<Double, Integer>> step = it.map(new Step()).split(new MySelector());
-
- // close the iteration by selecting the tuples that were directed to the
- // 'iterate' channel in the output selector
- it.closeWith(step.select("iterate"));
-
- // to produce the final output select the tuples directed to the
- // 'output' channel then project it to the desired second field
-
- DataStream<Tuple1<Integer>> numbers = step.select("output").project(1).types(Integer.class);
-
- // emit result
- if (fileOutput) {
- numbers.writeAsText(outputPath, 1);
- } else {
- numbers.print();
- }
-
- // execute the program
- env.execute("Streaming Iteration Example");
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /**
- * Iteration step function which takes an input (Double , Integer) and
- * produces an output (Double + random, Integer + 1).
- */
- public static class Step extends
- RichMapFunction<Tuple2<Double, Integer>, Tuple2<Double, Integer>> {
- private static final long serialVersionUID = 1L;
- private transient Random rnd;
-
- public void open(Configuration parameters) {
- rnd = new Random();
- }
-
- @Override
- public Tuple2<Double, Integer> map(Tuple2<Double, Integer> value) throws Exception {
- return new Tuple2<Double, Integer>(value.f0 + rnd.nextDouble(), value.f1 + 1);
- }
- }
-
- /**
- * OutputSelector testing which tuple needs to be iterated again.
- */
- public static class MySelector implements OutputSelector<Tuple2<Double, Integer>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Iterable<String> select(Tuple2<Double, Integer> value) {
- List<String> output = new ArrayList<String>();
- if (value.f0 > 100) {
- output.add("output");
- } else {
- output.add("iterate");
- }
- return output;
- }
-
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- // parse input arguments
- fileOutput = true;
- if (args.length == 1) {
- outputPath = args[0];
- } else {
- System.err.println("Usage: IterateExample <result path>");
- return false;
- }
- } else {
- System.out.println("Executing IterateExample with generated data.");
- System.out.println(" Provide parameter to write to file.");
- System.out.println(" Usage: IterateExample <result path>");
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
deleted file mode 100644
index dcfed50..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.join;
-
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.source.RichSourceFunction;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.util.Collector;
-
-/**
- * Example illustrating join over sliding windows of streams in Flink.
- *
- * <p>
- * his example will join two streams with a sliding window. One which emits
- * grades and one which emits salaries of people.
- * </p>
- *
- * <p>
- * This example shows how to:
- * <ul>
- * <li>do windowed joins,
- * <li>use tuple data types,
- * <li>write a simple streaming program.
- */
-public class WindowJoin {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- // obtain execution environment
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // connect to the data sources for grades and salaries
- DataStream<Tuple2<String, Integer>> grades = env.addSource(new GradeSource());
- DataStream<Tuple2<String, Integer>> salaries = env.addSource(new SalarySource());
-
- // apply a temporal join over the two stream based on the names over one
- // second windows
- DataStream<Tuple3<String, Integer, Integer>> joinedStream = grades
- .join(salaries)
- .onWindow(1, TimeUnit.SECONDS)
- .where(0)
- .equalTo(0)
- .with(new MyJoinFunction());
-
- // emit result
- if (fileOutput) {
- joinedStream.writeAsText(outputPath, 1);
- } else {
- joinedStream.print();
- }
-
- // execute program
- env.execute("Windowed Join Example");
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- private final static String[] names = { "tom", "jerry", "alice", "bob", "john", "grace" };
- private final static int GRADE_COUNT = 5;
- private final static int SALARY_MAX = 10000;
- private final static int SLEEP_TIME = 10;
-
- /**
- * Continuously emit tuples with random names and integers (grades).
- */
- public static class GradeSource implements SourceFunction<Tuple2<String, Integer>> {
- private static final long serialVersionUID = 1L;
-
- private Random rand;
- private Tuple2<String, Integer> outTuple;
-
- public GradeSource() {
- rand = new Random();
- outTuple = new Tuple2<String, Integer>();
- }
-
- @Override
- public void invoke(Collector<Tuple2<String, Integer>> out) throws Exception {
- while (true) {
- outTuple.f0 = names[rand.nextInt(names.length)];
- outTuple.f1 = rand.nextInt(GRADE_COUNT) + 1;
- out.collect(outTuple);
- Thread.sleep(rand.nextInt(SLEEP_TIME) + 1);
- }
- }
- }
-
- /**
- * Continuously emit tuples with random names and integers (salaries).
- */
- public static class SalarySource extends RichSourceFunction<Tuple2<String, Integer>> {
- private static final long serialVersionUID = 1L;
-
- private transient Random rand;
- private transient Tuple2<String, Integer> outTuple;
-
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- rand = new Random();
- outTuple = new Tuple2<String, Integer>();
- }
-
- @Override
- public void invoke(Collector<Tuple2<String, Integer>> out) throws Exception {
- while (true) {
- outTuple.f0 = names[rand.nextInt(names.length)];
- outTuple.f1 = rand.nextInt(SALARY_MAX) + 1;
- out.collect(outTuple);
- Thread.sleep(rand.nextInt(SLEEP_TIME) + 1);
- }
- }
- }
-
- public static class MyJoinFunction
- implements
- JoinFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Integer, Integer>> {
-
- private static final long serialVersionUID = 1L;
-
- private Tuple3<String, Integer, Integer> joined = new Tuple3<String, Integer, Integer>();
-
- @Override
- public Tuple3<String, Integer, Integer> join(Tuple2<String, Integer> first,
- Tuple2<String, Integer> second) throws Exception {
- joined.f0 = first.f0;
- joined.f1 = first.f1;
- joined.f2 = second.f1;
- return joined;
- }
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- // parse input arguments
- fileOutput = true;
- if (args.length == 1) {
- outputPath = args[0];
- } else {
- System.err.println("Usage: WindowJoin <result path>");
- return false;
- }
- } else {
- System.out.println("Executing WindowJoin with generated data.");
- System.out.println(" Provide parameter to write to file.");
- System.out.println(" Usage: WindowJoin <result path>");
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
deleted file mode 100755
index 375c86d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.ml;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.co.CoMapFunction;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.windowing.helper.Time;
-import org.apache.flink.util.Collector;
-
-/**
- * Skeleton for incremental machine learning algorithm consisting of a
- * pre-computed model, which gets updated for the new inputs and new input data
- * for which the job provides predictions.
- *
- * <p>
- * This may serve as a base of a number of algorithms, e.g. updating an
- * incremental Alternating Least Squares model while also providing the
- * predictions.
- * </p>
- *
- * <p>
- * This example shows how to use:
- * <ul>
- * <li>Connected streams
- * <li>CoFunctions
- * <li>Tuple data types
- * </ul>
- */
-public class IncrementalLearningSkeleton {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // build new model on every second of new data
- DataStream<Double[]> model = env.addSource(new TrainingDataSource())
- .window(Time.of(5000, TimeUnit.MILLISECONDS))
- .reduceGroup(new PartialModelBuilder());
-
- // use partial model for prediction
- DataStream<Integer> prediction = env.addSource(new NewDataSource()).connect(model)
- .map(new Predictor());
-
- // emit result
- if (fileOutput) {
- prediction.writeAsText(outputPath, 1);
- } else {
- prediction.print();
- }
-
- // execute program
- env.execute("Streaming Incremental Learning");
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /**
- * Feeds new data for prediction. By default it is implemented as constantly
- * emitting the Integer 1 in a loop.
- */
- public static class NewDataSource implements SourceFunction<Integer> {
- private static final long serialVersionUID = 1L;
- private static final int NEW_DATA_SLEEP_TIME = 1000;
-
- @Override
- public void invoke(Collector<Integer> collector) throws Exception {
- while (true) {
- collector.collect(getNewData());
- }
- }
-
- private Integer getNewData() throws InterruptedException {
- Thread.sleep(NEW_DATA_SLEEP_TIME);
- return 1;
- }
- }
-
- /**
- * Feeds new training data for the partial model builder. By default it is
- * implemented as constantly emitting the Integer 1 in a loop.
- */
- public static class TrainingDataSource implements SourceFunction<Integer> {
- private static final long serialVersionUID = 1L;
- private static final int TRAINING_DATA_SLEEP_TIME = 10;
-
- @Override
- public void invoke(Collector<Integer> collector) throws Exception {
- while (true) {
- collector.collect(getTrainingData());
- }
-
- }
-
- private Integer getTrainingData() throws InterruptedException {
- Thread.sleep(TRAINING_DATA_SLEEP_TIME);
- return 1;
-
- }
- }
-
- /**
- * Builds up-to-date partial models on new training data.
- */
- public static class PartialModelBuilder implements GroupReduceFunction<Integer, Double[]> {
- private static final long serialVersionUID = 1L;
-
- protected Double[] buildPartialModel(Iterable<Integer> values) {
- return new Double[] { 1. };
- }
-
- @Override
- public void reduce(Iterable<Integer> values, Collector<Double[]> out) throws Exception {
- out.collect(buildPartialModel(values));
- }
- }
-
- /**
- * Creates prediction using the model produced in batch-processing and the
- * up-to-date partial model.
- *
- * <p>
- * By defaults emits the Integer 0 for every prediction and the Integer 1
- * for every model update.
- * </p>
- */
- public static class Predictor implements CoMapFunction<Integer, Double[], Integer> {
- private static final long serialVersionUID = 1L;
-
- Double[] batchModel = null;
- Double[] partialModel = null;
-
- @Override
- public Integer map1(Integer value) {
- // Return prediction
- return predict(value);
- }
-
- @Override
- public Integer map2(Double[] value) {
- // Update model
- partialModel = value;
- batchModel = getBatchModel();
- return 1;
- }
-
- // pulls model built with batch-job on the old training data
- protected Double[] getBatchModel() {
- return new Double[] { 0. };
- }
-
- // performs prediction using the two models
- protected Integer predict(Integer inTuple) {
- return 0;
- }
-
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- // parse input arguments
- fileOutput = true;
- if (args.length == 1) {
- outputPath = args[0];
- } else {
- System.err.println("Usage: IncrementalLearningSkeleton <result path>");
- return false;
- }
- } else {
- System.out.println("Executing IncrementalLearningSkeleton with generated data.");
- System.out.println(" Provide parameter to write to file.");
- System.out.println(" Usage: IncrementalLearningSkeleton <result path>");
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
deleted file mode 100644
index e9b60f4..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.socket;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.examples.wordcount.WordCount.Tokenizer;
-
-/**
- * This example shows an implementation of WordCount with data from a text
- * socket. To run the example make sure that the service providing the text data
- * is already up and running.
- *
- * <p>
- * To start an example socket text stream on your local machine run netcat from
- * a command line: <code>nc -lk 9999</code>, where the parameter specifies the
- * port number.
- *
- *
- * <p>
- * Usage:
- * <code>SocketTextStreamWordCount <hostname> <port> <result path></code>
- * <br>
- *
- * <p>
- * This example shows how to:
- * <ul>
- * <li>use StreamExecutionEnvironment.socketTextStream
- * <li>write a simple Flink program,
- * <li>write and use user-defined functions.
- * </ul>
- *
- * @see <a href="www.openbsd.org/cgi-bin/man.cgi?query=nc">netcat</a>
- */
-public class SocketTextStreamWordCount {
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- // set up the execution environment
- final StreamExecutionEnvironment env = StreamExecutionEnvironment
- .getExecutionEnvironment();
-
- // get input data
- DataStream<String> text = env.socketTextStream(hostName, port);
-
- DataStream<Tuple2<String, Integer>> counts =
- // split up the lines in pairs (2-tuples) containing: (word,1)
- text.flatMap(new Tokenizer())
- // group by the tuple field "0" and sum up tuple field "1"
- .groupBy(0)
- .sum(1);
-
- if (fileOutput) {
- counts.writeAsText(outputPath, 1);
- } else {
- counts.print();
- }
-
- // execute program
- env.execute("WordCount from SocketTextStream Example");
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String hostName;
- private static int port;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
-
- // parse input arguments
- if (args.length == 3) {
- fileOutput = true;
- hostName = args[0];
- port = Integer.valueOf(args[1]);
- outputPath = args[2];
- } else if (args.length == 2) {
- hostName = args[0];
- port = Integer.valueOf(args[1]);
- } else {
- System.err.println("Usage: SocketTextStreamWordCount <hostname> <port> [<output path>]");
- return false;
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java
deleted file mode 100644
index 1901475..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.twitter;
-
-import java.util.StringTokenizer;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
-import org.apache.flink.streaming.examples.twitter.util.TwitterStreamData;
-import org.apache.flink.util.Collector;
-import org.apache.sling.commons.json.JSONException;
-
-/**
- * Implements the "TwitterStream" program that computes a most used word
- * occurrence over JSON files in a streaming fashion.
- *
- * <p>
- * The input is a JSON text file with lines separated by newline characters.
- *
- * <p>
- * Usage: <code>TwitterStream <text path></code><br>
- * If no parameters are provided, the program is run with default data from
- * {@link TwitterStreamData}.
- *
- * <p>
- * This example shows how to:
- * <ul>
- * <li>acquire external data,
- * <li>use in-line defined functions,
- * <li>handle flattened stream inputs.
- * </ul>
- *
- */
-public class TwitterStream {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
- if (!parseParameters(args)) {
- return;
- }
-
- // set up the execution environment
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- env.setBufferTimeout(1000);
-
- // get input data
- DataStream<String> streamSource = getTextDataStream(env);
-
- DataStream<Tuple2<String, Integer>> tweets = streamSource
- // selecting English tweets and splitting to words
- .flatMap(new SelectEnglishAndTokenizeFlatMap())
- // returning (word, 1)
- .map(new MapFunction<String, Tuple2<String, Integer>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<String, Integer> map(String value) throws Exception {
- return new Tuple2<String, Integer>(value, 1);
- }
- })
- // group by words and sum their occurence
- .groupBy(0).sum(1)
- // select word with maximum occurence
- .flatMap(new SelectMaxOccurence());
-
- // emit result
- if (fileOutput) {
- tweets.writeAsText(outputPath, 1);
- } else {
- tweets.print();
- }
-
- // execute program
- env.execute("Twitter Streaming Example");
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /**
- * Makes sentences from English tweets.
- *
- * <p>
- * Implements a string tokenizer that splits sentences into words as a
- * user-defined FlatMapFunction. The function takes a line (String) and
- * splits it into multiple pairs in the form of "(word,1)" (Tuple2<String,
- * Integer>).
- * </p>
- */
- public static class SelectEnglishAndTokenizeFlatMap extends JSONParseFlatMap<String, String> {
- private static final long serialVersionUID = 1L;
-
- /**
- * Select the language from the incoming JSON text
- */
- @Override
- public void flatMap(String value, Collector<String> out) throws Exception {
- try {
- if (getString(value, "lang").equals("en")) {
- // message of tweet
- StringTokenizer tokenizer = new StringTokenizer(getString(value, "text"));
-
- // split the message
- while (tokenizer.hasMoreTokens()) {
- String result = tokenizer.nextToken().replaceAll("\\s*", "");
-
- if (result != null && !result.equals("")) {
- out.collect(result);
- }
- }
- }
- } catch (JSONException e) {
-
- }
- }
- }
-
- /**
- * Implements a user-defined FlatMapFunction that checks if the current
- * occurence is higher than the maximum occurence. If so, returns the word
- * and changes the maximum.
- */
- public static class SelectMaxOccurence implements
- FlatMapFunction<Tuple2<String, Integer>, Tuple2<String, Integer>> {
- private static final long serialVersionUID = 1L;
- private Integer maximum;
-
- public SelectMaxOccurence() {
- this.maximum = 0;
- }
-
- @Override
- public void flatMap(Tuple2<String, Integer> value, Collector<Tuple2<String, Integer>> out)
- throws Exception {
- if ((Integer) value.getField(1) >= maximum) {
- out.collect(value);
- maximum = (Integer) value.getField(1);
- }
- }
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String textPath;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
- if (args.length > 0) {
- // parse input arguments
- fileOutput = true;
- if (args.length == 2) {
- textPath = args[0];
- outputPath = args[1];
- } else {
- System.err.println("USAGE:\nTwitterStream <pathToPropertiesFile> <result path>");
- return false;
- }
- } else {
- System.out.println("Executing TwitterStream example with built-in default data.");
- System.out.println(" Provide parameters to read input data from a file.");
- System.out.println(" USAGE: TwitterStream <pathToPropertiesFile>");
- }
- return true;
- }
-
- private static DataStream<String> getTextDataStream(StreamExecutionEnvironment env) {
- if (fileOutput) {
- // read the text file from given input path
- return env.readTextFile(textPath);
- } else {
- // get default test text data
- return env.fromElements(TwitterStreamData.TEXTS);
- }
- }
-}
[36/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
deleted file mode 100644
index ed246c8..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-public class WindowReduceInvokable<IN> extends WindowInvokable<IN, IN> {
-
- private static final long serialVersionUID = 1L;
-
- ReduceFunction<IN> reducer;
-
- public WindowReduceInvokable(ReduceFunction<IN> userFunction,
- LinkedList<TriggerPolicy<IN>> triggerPolicies,
- LinkedList<EvictionPolicy<IN>> evictionPolicies) {
- super(userFunction, triggerPolicies, evictionPolicies);
- this.reducer = userFunction;
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- Iterator<IN> reducedIterator = buffer.iterator();
- IN reduced = null;
-
- while (reducedIterator.hasNext() && reduced == null) {
- reduced = reducedIterator.next();
- }
-
- while (reducedIterator.hasNext()) {
- IN next = reducedIterator.next();
- if (next != null) {
- reduced = reducer.reduce(copy(reduced), copy(next));
- }
- }
- if (reduced != null) {
- collector.collect(reduced);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapInvokable.java
deleted file mode 100644
index 4cbaebb..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapInvokable.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
-
-public class CoFlatMapInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
-
- private CoFlatMapFunction<IN1, IN2, OUT> flatMapper;
-
- public CoFlatMapInvokable(CoFlatMapFunction<IN1, IN2, OUT> flatMapper) {
- super(flatMapper);
- this.flatMapper = flatMapper;
- }
-
- @Override
- public void handleStream1() throws Exception {
- callUserFunctionAndLogException1();
- }
-
- @Override
- public void handleStream2() throws Exception {
- callUserFunctionAndLogException2();
- }
-
- @Override
- protected void callUserFunction1() throws Exception {
- flatMapper.flatMap1(reuse1.getObject(), collector);
-
- }
-
- @Override
- protected void callUserFunction2() throws Exception {
- flatMapper.flatMap2(reuse2.getObject(), collector);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceInvokable.java
deleted file mode 100644
index 4907ac5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceInvokable.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-
-public class CoGroupedReduceInvokable<IN1, IN2, OUT> extends CoReduceInvokable<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
-
- protected KeySelector<IN1, ?> keySelector1;
- protected KeySelector<IN2, ?> keySelector2;
- private Map<Object, IN1> values1;
- private Map<Object, IN2> values2;
- IN1 reduced1;
- IN2 reduced2;
-
- public CoGroupedReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer,
- KeySelector<IN1, ?> keySelector1, KeySelector<IN2, ?> keySelector2) {
- super(coReducer);
- this.coReducer = coReducer;
- this.keySelector1 = keySelector1;
- this.keySelector2 = keySelector2;
- values1 = new HashMap<Object, IN1>();
- values2 = new HashMap<Object, IN2>();
- }
-
- @Override
- public void handleStream1() throws Exception {
- Object key = reuse1.getKey(keySelector1);
- currentValue1 = values1.get(key);
- nextValue1 = reuse1.getObject();
- if (currentValue1 != null) {
- callUserFunctionAndLogException1();
- values1.put(key, reduced1);
- collector.collect(coReducer.map1(reduced1));
- } else {
- values1.put(key, nextValue1);
- collector.collect(coReducer.map1(nextValue1));
- }
- }
-
- @Override
- public void handleStream2() throws Exception {
- Object key = reuse2.getKey(keySelector2);
- currentValue2 = values2.get(key);
- nextValue2 = reuse2.getObject();
- if (currentValue2 != null) {
- callUserFunctionAndLogException2();
- values2.put(key, reduced2);
- collector.collect(coReducer.map2(reduced2));
- } else {
- values2.put(key, nextValue2);
- collector.collect(coReducer.map2(nextValue2));
- }
- }
-
- @Override
- protected void callUserFunction1() throws Exception {
- reduced1 = coReducer.reduce1(currentValue1, nextValue1);
-
- }
-
- @Override
- protected void callUserFunction2() throws Exception {
- reduced2 = coReducer.reduce2(currentValue2, nextValue2);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java
deleted file mode 100644
index 604873e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.api.streamvertex.StreamTaskContext;
-import org.apache.flink.streaming.io.CoReaderIterator;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class CoInvokable<IN1, IN2, OUT> extends StreamInvokable<IN1, OUT> {
-
- public CoInvokable(Function userFunction) {
- super(userFunction);
- }
-
- private static final long serialVersionUID = 1L;
- private static final Logger LOG = LoggerFactory.getLogger(CoInvokable.class);
-
- protected CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>> recordIterator;
- protected StreamRecord<IN1> reuse1;
- protected StreamRecord<IN2> reuse2;
- protected StreamRecordSerializer<IN1> srSerializer1;
- protected StreamRecordSerializer<IN2> srSerializer2;
- protected TypeSerializer<IN1> serializer1;
- protected TypeSerializer<IN2> serializer2;
-
- @Override
- public void setup(StreamTaskContext<OUT> taskContext) {
- this.collector = taskContext.getOutputCollector();
-
- this.recordIterator = taskContext.getCoReader();
-
- this.srSerializer1 = taskContext.getInputSerializer(0);
- this.srSerializer2 = taskContext.getInputSerializer(1);
-
- this.reuse1 = srSerializer1.createInstance();
- this.reuse2 = srSerializer2.createInstance();
-
- this.serializer1 = srSerializer1.getObjectSerializer();
- this.serializer2 = srSerializer2.getObjectSerializer();
- }
-
- protected void resetReuseAll() {
- this.reuse1 = srSerializer1.createInstance();
- this.reuse2 = srSerializer2.createInstance();
- }
-
- protected void resetReuse1() {
- this.reuse1 = srSerializer1.createInstance();
- }
-
- protected void resetReuse2() {
- this.reuse2 = srSerializer2.createInstance();
- }
-
- @Override
- public void invoke() throws Exception {
- while (true) {
- int next = recordIterator.next(reuse1, reuse2);
- if (next == 0) {
- break;
- } else if (next == 1) {
- initialize1();
- handleStream1();
- resetReuse1();
- } else {
- initialize2();
- handleStream2();
- resetReuse2();
- }
- }
- }
-
- protected abstract void handleStream1() throws Exception;
-
- protected abstract void handleStream2() throws Exception;
-
- protected abstract void callUserFunction1() throws Exception;
-
- protected abstract void callUserFunction2() throws Exception;
-
- protected void initialize1() {
-
- };
-
- protected void initialize2() {
-
- };
-
- protected void callUserFunctionAndLogException1() {
- try {
- callUserFunction1();
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Calling user function failed due to: {}",
- StringUtils.stringifyException(e));
- }
- }
- }
-
- protected void callUserFunctionAndLogException2() {
- try {
- callUserFunction2();
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Calling user function failed due to: {}",
- StringUtils.stringifyException(e));
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapInvokable.java
deleted file mode 100644
index 5499dba..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapInvokable.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import org.apache.flink.streaming.api.function.co.CoMapFunction;
-
-public class CoMapInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
-
- private CoMapFunction<IN1, IN2, OUT> mapper;
-
- public CoMapInvokable(CoMapFunction<IN1, IN2, OUT> mapper) {
- super(mapper);
- this.mapper = mapper;
- }
-
- @Override
- public void handleStream1() throws Exception {
- callUserFunctionAndLogException1();
- }
-
- @Override
- public void handleStream2() throws Exception {
- callUserFunctionAndLogException2();
- }
-
- @Override
- protected void callUserFunction1() throws Exception {
- collector.collect(mapper.map1(reuse1.getObject()));
-
- }
-
- @Override
- protected void callUserFunction2() throws Exception {
- collector.collect(mapper.map2(reuse2.getObject()));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java
deleted file mode 100644
index 057dfce..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-
-public class CoReduceInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
-
- protected CoReduceFunction<IN1, IN2, OUT> coReducer;
- protected IN1 currentValue1 = null;
- protected IN2 currentValue2 = null;
- protected IN1 nextValue1 = null;
- protected IN2 nextValue2 = null;
-
- public CoReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer) {
- super(coReducer);
- this.coReducer = coReducer;
- currentValue1 = null;
- currentValue2 = null;
- }
-
- @Override
- public void handleStream1() throws Exception {
- nextValue1 = reuse1.getObject();
- callUserFunctionAndLogException1();
- }
-
- @Override
- public void handleStream2() throws Exception {
- nextValue2 = reuse2.getObject();
- callUserFunctionAndLogException2();
- }
-
- @Override
- protected void callUserFunction1() throws Exception {
- if (currentValue1 != null) {
- currentValue1 = coReducer.reduce1(currentValue1, nextValue1);
- } else {
- currentValue1 = nextValue1;
- }
- collector.collect(coReducer.map1(currentValue1));
- }
-
- @Override
- protected void callUserFunction2() throws Exception {
- if (currentValue2 != null) {
- currentValue2 = coReducer.reduce2(currentValue2, nextValue2);
- } else {
- currentValue2 = nextValue2;
- }
- collector.collect(coReducer.map2(currentValue2));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
deleted file mode 100644
index 93f597f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.math.util.MathUtils;
-import org.apache.flink.streaming.api.function.co.CoWindowFunction;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.state.CircularFifoList;
-
-public class CoWindowInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
-
- protected CoWindowFunction<IN1, IN2, OUT> coWindowFunction;
- protected long windowSize;
- protected long slideSize;
- protected CircularFifoList<StreamRecord<IN1>> circularList1;
- protected CircularFifoList<StreamRecord<IN2>> circularList2;
- protected TimestampWrapper<IN1> timeStamp1;
- protected TimestampWrapper<IN2> timeStamp2;
-
- protected StreamWindow window;
-
- protected long startTime;
- protected long nextRecordTime;
-
- public CoWindowInvokable(CoWindowFunction<IN1, IN2, OUT> coWindowFunction, long windowSize,
- long slideInterval, TimestampWrapper<IN1> timeStamp1, TimestampWrapper<IN2> timeStamp2) {
- super(coWindowFunction);
- this.coWindowFunction = coWindowFunction;
- this.windowSize = windowSize;
- this.slideSize = slideInterval;
- this.circularList1 = new CircularFifoList<StreamRecord<IN1>>();
- this.circularList2 = new CircularFifoList<StreamRecord<IN2>>();
- this.timeStamp1 = timeStamp1;
- this.timeStamp2 = timeStamp2;
- this.startTime = timeStamp1.getStartTime();
-
- this.window = new StreamWindow();
- }
-
- @Override
- protected void handleStream1() throws Exception {
- window.addToBuffer1(reuse1.getObject());
- }
-
- @Override
- protected void handleStream2() throws Exception {
- window.addToBuffer2(reuse2.getObject());
- }
-
- @Override
- protected void callUserFunction() throws Exception {
-
- List<IN1> first = new ArrayList<IN1>();
- List<IN2> second = new ArrayList<IN2>();
-
- for (IN1 element : window.circularList1.getElements()) {
- first.add(serializer1.copy(element));
- }
- for (IN2 element : window.circularList2.getElements()) {
- second.add(serializer2.copy(element));
- }
-
- if (!window.circularList1.isEmpty() || !window.circularList2.isEmpty()) {
- coWindowFunction.coWindow(first, second, collector);
- }
- }
-
- protected class StreamWindow implements Serializable {
- private static final long serialVersionUID = 1L;
-
- protected int granularity;
- protected int batchPerSlide;
- protected long numberOfBatches;
-
- protected long minibatchCounter;
-
- protected CircularFifoList<IN1> circularList1;
- protected CircularFifoList<IN2> circularList2;
-
- public StreamWindow() {
- this.granularity = (int) MathUtils.gcd(windowSize, slideSize);
- this.batchPerSlide = (int) (slideSize / granularity);
- this.numberOfBatches = windowSize / granularity;
- this.circularList1 = new CircularFifoList<IN1>();
- this.circularList2 = new CircularFifoList<IN2>();
- this.minibatchCounter = 0;
- }
-
- public void addToBuffer1(IN1 nextValue) throws Exception {
- checkWindowEnd(timeStamp1.getTimestamp(nextValue));
- if (minibatchCounter >= 0) {
- circularList1.add(nextValue);
- }
- }
-
- public void addToBuffer2(IN2 nextValue) throws Exception {
- checkWindowEnd(timeStamp2.getTimestamp(nextValue));
- if (minibatchCounter >= 0) {
- circularList2.add(nextValue);
- }
- }
-
- protected synchronized void checkWindowEnd(long timeStamp) {
- nextRecordTime = timeStamp;
-
- while (miniBatchEnd()) {
- circularList1.newSlide();
- circularList2.newSlide();
- minibatchCounter++;
- if (windowEnd()) {
- callUserFunctionAndLogException();
- circularList1.shiftWindow(batchPerSlide);
- circularList2.shiftWindow(batchPerSlide);
- }
- }
- }
-
- protected boolean miniBatchEnd() {
- if (nextRecordTime < startTime + granularity) {
- return false;
- } else {
- startTime += granularity;
- return true;
- }
- }
-
- public boolean windowEnd() {
- if (minibatchCounter == numberOfBatches) {
- minibatchCounter -= batchPerSlide;
- return true;
- }
- return false;
- }
-
- public void reduceLastBatch() {
- if (!miniBatchEnd()) {
- callUserFunctionAndLogException();
- }
- }
-
- public Iterable<IN1> getIterable1() {
- return circularList1.getIterable();
- }
-
- public Iterable<IN2> getIterable2() {
- return circularList2.getIterable();
- }
-
- @Override
- public String toString() {
- return circularList1.toString();
- }
-
- }
-
- @Override
- public void close() {
- if (!window.miniBatchEnd()) {
- callUserFunctionAndLogException();
- }
- super.close();
- }
-
- @Override
- protected void callUserFunction1() throws Exception {
- }
-
- @Override
- protected void callUserFunction2() throws Exception {
- }
-
- public void setSlideSize(long slideSize) {
- this.slideSize = slideSize;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecord.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecord.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecord.java
deleted file mode 100644
index 76f37be..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecord.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamrecord;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple;
-
-/**
- * Object for wrapping a tuple or other object with ID used for sending records
- * between streaming task in Apache Flink stream processing.
- */
-public class StreamRecord<T> implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private UID uid;
- private T streamObject;
- public boolean isTuple;
-
- /**
- * Creates an empty StreamRecord
- */
- public StreamRecord() {
- uid = new UID();
- }
-
- /**
- * @return The ID of the object
- */
- public UID getId() {
- return uid;
- }
-
- /**
- * Creates a new ID for the StreamRecord using the given channelID
- *
- * @param channelID
- * ID of the emitting task
- * @return The StreamRecord object
- */
- public StreamRecord<T> newId(int channelID) {
- uid = new UID(channelID);
- return this;
- }
-
- /**
- * Sets the ID of the StreamRecord
- *
- * @param id
- * id to set
- */
- public void setId(UID id) {
- this.uid = id;
- }
-
- /**
- * Gets the wrapped object from the StreamRecord
- *
- * @return The object wrapped
- */
- public T getObject() {
- return streamObject;
- }
-
- /**
- * Gets the field of the contained object at the given position. If a tuple
- * is wrapped then the getField method is invoked. If the StreamRecord
- * contains and object of Basic types only position 0 could be returned.
- *
- * @param pos
- * Position of the field to get.
- * @return Returns the object contained in the position.
- */
- public Object getField(int pos) {
- if (isTuple) {
- return ((Tuple) streamObject).getField(pos);
- } else {
- if (pos == 0) {
- return streamObject;
- } else {
- throw new IndexOutOfBoundsException();
- }
- }
- }
-
- /**
- * Extracts key for the stored object using the keySelector provided.
- *
- * @param keySelector
- * KeySelector for extracting the key
- * @return The extracted key
- */
- public <R> R getKey(KeySelector<T, R> keySelector) {
- try {
- return keySelector.getKey(streamObject);
- } catch (Exception e) {
- throw new RuntimeException("Failed to extract key: " + e.getMessage());
- }
- }
-
- /**
- * Sets the object stored
- *
- * @param object
- * Object to set
- * @return Returns the StreamRecord object
- */
- public StreamRecord<T> setObject(T object) {
- this.streamObject = object;
- return this;
- }
-
- @Override
- public String toString() {
- return streamObject.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecordSerializer.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecordSerializer.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecordSerializer.java
deleted file mode 100755
index 85faa9e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/StreamRecordSerializer.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamrecord;
-
-import java.io.IOException;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-
-public final class StreamRecordSerializer<T> extends TypeSerializer<StreamRecord<T>> {
-
- private static final long serialVersionUID = 1L;
-
- private final TypeSerializer<T> typeSerializer;
- private final boolean isTuple;
-
- public StreamRecordSerializer(TypeInformation<T> typeInfo) {
- this.typeSerializer = typeInfo.createSerializer();
- this.isTuple = typeInfo.isTupleType();
- }
-
- public TypeSerializer<T> getObjectSerializer() {
- return typeSerializer;
- }
-
- @Override
- public boolean isImmutableType() {
- return false;
- }
-
- @Override
- public boolean isStateful() {
- return false;
- }
-
- @Override
- public StreamRecord<T> createInstance() {
- try {
- StreamRecord<T> t = new StreamRecord<T>();
- t.isTuple = isTuple;
- t.setObject(typeSerializer.createInstance());
- return t;
- } catch (Exception e) {
- throw new RuntimeException("Cannot instantiate StreamRecord.", e);
- }
- }
-
- @Override
- public StreamRecord<T> copy(StreamRecord<T> from) {
- StreamRecord<T> rec = new StreamRecord<T>();
- rec.isTuple = from.isTuple;
- rec.setId(from.getId().copy());
- rec.setObject(typeSerializer.copy(from.getObject()));
- return rec;
- }
-
- @Override
- public StreamRecord<T> copy(StreamRecord<T> from, StreamRecord<T> reuse) {
- reuse.isTuple = from.isTuple;
- reuse.setId(from.getId().copy());
- reuse.setObject(typeSerializer.copy(from.getObject(), reuse.getObject()));
- return reuse;
- }
-
- @Override
- public int getLength() {
- return -1;
- }
-
- @Override
- public void serialize(StreamRecord<T> value, DataOutputView target) throws IOException {
- value.getId().write(target);
- typeSerializer.serialize(value.getObject(), target);
- }
-
- @Override
- public StreamRecord<T> deserialize(DataInputView source) throws IOException {
- StreamRecord<T> record = new StreamRecord<T>();
- record.isTuple = this.isTuple;
- record.getId().read(source);
- record.setObject(typeSerializer.deserialize(source));
- return record;
- }
-
- @Override
- public StreamRecord<T> deserialize(StreamRecord<T> reuse, DataInputView source) throws IOException {
- reuse.getId().read(source);
- reuse.setObject(typeSerializer.deserialize(reuse.getObject(), source));
- return reuse;
- }
-
- @Override
- public void copy(DataInputView source, DataOutputView target) throws IOException {
- // Needs to be implemented
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/UID.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/UID.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/UID.java
deleted file mode 100644
index b79b7e5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamrecord/UID.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamrecord;
-
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Random;
-
-import org.apache.flink.core.io.IOReadableWritable;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-
-/**
- * Object for creating unique IDs for {@link StreamRecord}s.
- *
- **/
-public class UID implements IOReadableWritable, Serializable {
- private static final long serialVersionUID = 1L;
-
- private ByteBuffer uid;
- private static Random random = new Random();
-
- public UID() {
- uid = ByteBuffer.allocate(20);
- }
-
- // TODO: consider sequential ids
- public UID(int channelID) {
- byte[] uuid = new byte[16];
- random.nextBytes(uuid);
- uid = ByteBuffer.allocate(20).putInt(channelID).put(uuid);
- }
-
- UID(byte[] id) {
- uid = ByteBuffer.wrap(id);
- }
-
- public int getChannelId() {
- uid.position(0);
- return uid.getInt();
- }
-
- public byte[] getGeneratedId() {
- uid.position(4);
- return uid.slice().array();
- }
-
- public byte[] getId() {
- uid.position(0);
- return uid.array();
- }
-
- @Override
- public void write(DataOutputView out) throws IOException {
- out.write(uid.array());
- }
-
- private void writeObject(ObjectOutputStream stream) throws IOException {
- stream.write(uid.array());
- }
-
- private void readObject(java.io.ObjectInputStream stream) throws IOException,
- ClassNotFoundException {
- byte[] uidA = new byte[20];
- stream.read(uidA);
- uid = ByteBuffer.allocate(20).put(uidA);
- }
-
- @Override
- public void read(DataInputView in) throws IOException {
- byte[] uidByteArray = new byte[20];
- in.readFully(uidByteArray, 0, 20);
- uid = ByteBuffer.wrap(uidByteArray);
- }
-
- @Override
- public String toString() {
- return getChannelId() + "-" + Long.toHexString(uid.getLong(4)) + "-"
- + Long.toHexString(uid.getLong(12));
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(getId());
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- } else {
- try {
- UID other = (UID) obj;
- return Arrays.equals(this.getId(), other.getId());
- } catch (ClassCastException e) {
- return false;
- }
- }
- }
-
- public UID copy() {
- return new UID(Arrays.copyOf(uid.array(), 20));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
deleted file mode 100644
index 2b650be..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-import org.apache.flink.runtime.io.network.api.reader.BufferReader;
-import org.apache.flink.runtime.io.network.api.reader.BufferReaderBase;
-import org.apache.flink.runtime.io.network.api.reader.UnionBufferReader;
-import org.apache.flink.runtime.plugable.DeserializationDelegate;
-import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.io.CoReaderIterator;
-import org.apache.flink.streaming.io.CoRecordReader;
-import org.apache.flink.util.MutableObjectIterator;
-
-import java.util.ArrayList;
-
-public class CoStreamVertex<IN1, IN2, OUT> extends StreamVertex<IN1, OUT> {
-
- protected StreamRecordSerializer<IN1> inputDeserializer1 = null;
- protected StreamRecordSerializer<IN2> inputDeserializer2 = null;
-
- MutableObjectIterator<StreamRecord<IN1>> inputIter1;
- MutableObjectIterator<StreamRecord<IN2>> inputIter2;
-
- CoRecordReader<DeserializationDelegate<StreamRecord<IN1>>, DeserializationDelegate<StreamRecord<IN2>>> coReader;
- CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>> coIter;
-
- private CoInvokable<IN1, IN2, OUT> userInvokable;
- private static int numTasks;
-
- public CoStreamVertex() {
- userInvokable = null;
- numTasks = newVertex();
- instanceID = numTasks;
- }
-
- private void setDeserializers() {
- inputDeserializer1 = configuration.getTypeSerializerIn1(userClassLoader);
- inputDeserializer2 = configuration.getTypeSerializerIn2(userClassLoader);
- }
-
- @Override
- public void setInputsOutputs() {
- outputHandler = new OutputHandler<OUT>(this);
-
- setConfigInputs();
-
- coIter = new CoReaderIterator<StreamRecord<IN1>, StreamRecord<IN2>>(coReader,
- inputDeserializer1, inputDeserializer2);
- }
-
- @Override
- protected void setInvokable() {
- userInvokable = configuration.getUserInvokable(userClassLoader);
- userInvokable.setup(this);
- }
-
- protected void setConfigInputs() throws StreamVertexException {
- setDeserializers();
-
- int numberOfInputs = configuration.getNumberOfInputs();
-
- ArrayList<BufferReader> inputList1 = new ArrayList<BufferReader>();
- ArrayList<BufferReader> inputList2 = new ArrayList<BufferReader>();
-
- for (int i = 0; i < numberOfInputs; i++) {
- int inputType = configuration.getInputIndex(i);
- BufferReader reader = getEnvironment().getReader(i);
- switch (inputType) {
- case 1:
- inputList1.add(reader);
- break;
- case 2:
- inputList2.add(reader);
- break;
- default:
- throw new RuntimeException("Invalid input type number: " + inputType);
- }
- }
-
- final BufferReaderBase reader1 = inputList1.size() == 1
- ? inputList1.get(0)
- : new UnionBufferReader(inputList1.toArray(new BufferReader[inputList1.size()]));
-
- final BufferReaderBase reader2 = inputList2.size() == 1
- ? inputList2.get(0)
- : new UnionBufferReader(inputList2.toArray(new BufferReader[inputList2.size()]));
-
- coReader = new CoRecordReader<DeserializationDelegate<StreamRecord<IN1>>, DeserializationDelegate<StreamRecord<IN2>>>(reader1, reader2);
- }
-
- @Override
- public void invoke() throws Exception {
- outputHandler.invokeUserFunction("CO-TASK", userInvokable);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <X> MutableObjectIterator<X> getInput(int index) {
- switch (index) {
- case 0:
- return (MutableObjectIterator<X>) inputIter1;
- case 1:
- return (MutableObjectIterator<X>) inputIter2;
- default:
- throw new IllegalArgumentException("CoStreamVertex has only 2 inputs");
- }
- }
- @SuppressWarnings("unchecked")
- @Override
- public <X> StreamRecordSerializer<X> getInputSerializer(int index) {
- switch (index) {
- case 0:
- return (StreamRecordSerializer<X>) inputDeserializer1;
- case 1:
- return (StreamRecordSerializer<X>) inputDeserializer2;
- default:
- throw new IllegalArgumentException("CoStreamVertex has only 2 inputs");
- }
- }
- @SuppressWarnings("unchecked")
- @Override
- public <X, Y> CoReaderIterator<X, Y> getCoReader() {
- return (CoReaderIterator<X, Y>) coIter;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
deleted file mode 100644
index ad6d948..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.io.IOReadableWritable;
-import org.apache.flink.runtime.io.network.api.reader.MutableReader;
-import org.apache.flink.runtime.io.network.api.reader.MutableRecordReader;
-import org.apache.flink.runtime.io.network.api.reader.UnionBufferReader;
-import org.apache.flink.runtime.operators.util.ReaderIterator;
-import org.apache.flink.runtime.plugable.DeserializationDelegate;
-import org.apache.flink.streaming.api.StreamConfig;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.util.MutableObjectIterator;
-
-public class InputHandler<IN> {
- private StreamRecordSerializer<IN> inputSerializer = null;
- private MutableObjectIterator<StreamRecord<IN>> inputIter;
- private MutableReader<IOReadableWritable> inputs;
-
- private StreamVertex<IN, ?> streamVertex;
- private StreamConfig configuration;
-
- public InputHandler(StreamVertex<IN, ?> streamComponent) {
- this.streamVertex = streamComponent;
- this.configuration = new StreamConfig(streamComponent.getTaskConfiguration());
- try {
- setConfigInputs();
- } catch (Exception e) {
- throw new StreamVertexException("Cannot register inputs for "
- + getClass().getSimpleName(), e);
- }
-
- }
-
- protected void setConfigInputs() throws StreamVertexException {
- inputSerializer = configuration.getTypeSerializerIn1(streamVertex.userClassLoader);
-
- int numberOfInputs = configuration.getNumberOfInputs();
- if (numberOfInputs > 0) {
-
- if (numberOfInputs < 2) {
-
- inputs = new MutableRecordReader<IOReadableWritable>(streamVertex.getEnvironment().getReader(0));
-
- } else {
- UnionBufferReader reader = new UnionBufferReader(streamVertex.getEnvironment().getAllReaders());
- inputs = new MutableRecordReader<IOReadableWritable>(reader);
- }
-
- inputIter = createInputIterator();
- }
- }
-
- private MutableObjectIterator<StreamRecord<IN>> createInputIterator() {
- @SuppressWarnings({ "unchecked", "rawtypes" })
- final MutableObjectIterator<StreamRecord<IN>> iter = new ReaderIterator(inputs,
- inputSerializer);
- return iter;
- }
-
- protected static <T> MutableObjectIterator<StreamRecord<T>> staticCreateInputIterator(
- MutableReader<?> inputReader, TypeSerializer<?> serializer) {
-
- // generic data type serialization
- @SuppressWarnings("unchecked")
- MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
- @SuppressWarnings({ "unchecked", "rawtypes" })
- final MutableObjectIterator<StreamRecord<T>> iter = new ReaderIterator(reader, serializer);
- return iter;
- }
-
- public StreamRecordSerializer<IN> getInputSerializer() {
- return inputSerializer;
- }
-
- public MutableObjectIterator<StreamRecord<IN>> getInputIter() {
- return inputIter;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
deleted file mode 100644
index 1a12cb2..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.StreamConfig;
-import org.apache.flink.streaming.api.collector.CollectorWrapper;
-import org.apache.flink.streaming.api.collector.DirectedCollectorWrapper;
-import org.apache.flink.streaming.api.collector.StreamOutput;
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.io.StreamRecordWriter;
-import org.apache.flink.streaming.partitioner.StreamPartitioner;
-import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class OutputHandler<OUT> {
- private static final Logger LOG = LoggerFactory.getLogger(OutputHandler.class);
-
- private StreamVertex<?, OUT> vertex;
- private StreamConfig configuration;
- private ClassLoader cl;
- private Collector<OUT> outerCollector;
-
- public List<ChainableInvokable<?, ?>> chainedInvokables;
-
- private Map<String, StreamOutput<?>> outputMap;
- private Map<String, StreamConfig> chainedConfigs;
- private List<Tuple2<String, String>> outEdgesInOrder;
-
- public OutputHandler(StreamVertex<?, OUT> vertex) {
-
- // Initialize some fields
- this.vertex = vertex;
- this.configuration = new StreamConfig(vertex.getTaskConfiguration());
- this.chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
- this.outputMap = new HashMap<String, StreamOutput<?>>();
- this.cl = vertex.getUserCodeClassLoader();
-
- // We read the chained configs, and the order of record writer
- // registrations by outputname
- this.chainedConfigs = configuration.getTransitiveChainedTaskConfigs(cl);
- this.chainedConfigs.put(configuration.getTaskName(), configuration);
-
- this.outEdgesInOrder = configuration.getOutEdgesInOrder(cl);
-
- // We iterate through all the out edges from this job vertex and create
- // a stream output
- for (Tuple2<String, String> outEdge : outEdgesInOrder) {
- StreamOutput<?> streamOutput = createStreamOutput(outEdge.f1,
- chainedConfigs.get(outEdge.f0), outEdgesInOrder.indexOf(outEdge));
- outputMap.put(outEdge.f1, streamOutput);
- }
-
- // We create the outer collector that will be passed to the first task
- // in the chain
- this.outerCollector = createChainedCollector(configuration);
-
- }
-
- public Collection<StreamOutput<?>> getOutputs() {
- return outputMap.values();
- }
-
- /**
- * This method builds up a nested collector which encapsulates all the
- * chained operators and their network output. The result of this recursive
- * call will be passed as collector to the first invokable in the chain.
- *
- * @param chainedTaskConfig
- * The configuration of the starting operator of the chain, we
- * use this paramater to recursively build the whole chain
- * @return Returns the collector for the chain starting from the given
- * config
- */
- @SuppressWarnings({ "unchecked", "rawtypes" })
- private Collector<OUT> createChainedCollector(StreamConfig chainedTaskConfig) {
-
- boolean isDirectEmit = chainedTaskConfig.isDirectedEmit();
-
- // We create a wrapper that will encapsulate the chained operators and
- // network outputs
- CollectorWrapper<OUT> wrapper = isDirectEmit ? new DirectedCollectorWrapper(
- chainedTaskConfig.getOutputSelectors(cl)) : new CollectorWrapper<OUT>();
-
- // Create collectors for the network outputs
- for (String output : chainedTaskConfig.getOutputs(cl)) {
-
- Collector<?> outCollector = outputMap.get(output);
-
- if (isDirectEmit) {
- ((DirectedCollectorWrapper<OUT>) wrapper).addCollector(outCollector,
- chainedTaskConfig.getSelectedNames(output));
- } else {
- wrapper.addCollector(outCollector);
- }
- }
-
- // Create collectors for the chained outputs
- for (String output : chainedTaskConfig.getChainedOutputs(cl)) {
- Collector<?> outCollector = createChainedCollector(chainedConfigs.get(output));
- if (isDirectEmit) {
- ((DirectedCollectorWrapper<OUT>) wrapper).addCollector(outCollector,
- chainedTaskConfig.getSelectedNames(output));
- } else {
- wrapper.addCollector(outCollector);
- }
- }
-
- if (chainedTaskConfig.isChainStart()) {
- // The current task is the first chained task at this vertex so we
- // return the wrapper
- return wrapper;
- } else {
- // The current task is a part of the chain so we get the chainable
- // invokable which will be returned and set it up using the wrapper
- ChainableInvokable chainableInvokable = chainedTaskConfig.getUserInvokable(vertex
- .getUserCodeClassLoader());
- chainableInvokable.setup(wrapper,
- chainedTaskConfig.getTypeSerializerIn1(vertex.getUserCodeClassLoader()));
-
- chainedInvokables.add(chainableInvokable);
- return chainableInvokable;
- }
-
- }
-
- public Collector<OUT> getCollector() {
- return outerCollector;
- }
-
- /**
- * We create the StreamOutput for the specific output given by the name, and
- * the configuration of its source task
- *
- * @param outputVertex
- * Name of the output to which the streamoutput will be set up
- * @param configuration
- * The config of upStream task
- * @return
- */
- private <T> StreamOutput<T> createStreamOutput(String outputVertex, StreamConfig configuration,
- int outputIndex) {
-
- StreamRecordSerializer<T> outSerializer = configuration
- .getTypeSerializerOut1(vertex.userClassLoader);
- SerializationDelegate<StreamRecord<T>> outSerializationDelegate = null;
-
- if (outSerializer != null) {
- outSerializationDelegate = new SerializationDelegate<StreamRecord<T>>(outSerializer);
- outSerializationDelegate.setInstance(outSerializer.createInstance());
- }
-
- StreamPartitioner<T> outputPartitioner = configuration.getPartitioner(cl, outputVertex);
-
- RecordWriter<SerializationDelegate<StreamRecord<T>>> output;
-
- if (configuration.getBufferTimeout() >= 0) {
-
- output = new StreamRecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
- .getEnvironment().getWriter(outputIndex), outputPartitioner,
- configuration.getBufferTimeout());
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("StreamRecordWriter initiated with {} bufferTimeout for {}",
- configuration.getBufferTimeout(), vertex.getClass().getSimpleName());
- }
- } else {
- output = new RecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
- .getEnvironment().getWriter(outputIndex), outputPartitioner);
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("RecordWriter initiated for {}", vertex.getClass().getSimpleName());
- }
- }
-
- StreamOutput<T> streamOutput = new StreamOutput<T>(output, vertex.instanceID,
- outSerializationDelegate);
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
- .getSimpleName(), outputIndex, vertex.getClass().getSimpleName());
- }
-
- return streamOutput;
- }
-
- public void flushOutputs() throws IOException, InterruptedException {
- for (StreamOutput<?> streamOutput : getOutputs()) {
- streamOutput.close();
- }
- }
-
- public void invokeUserFunction(String componentTypeName, StreamInvokable<?, OUT> userInvokable)
- throws IOException, InterruptedException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("{} {} invoked with instance id {}", componentTypeName, vertex.getName(),
- vertex.getInstanceID());
- }
-
- try {
- vertex.invokeUserFunction(userInvokable);
- } catch (Exception e) {
- flushOutputs();
- throw new RuntimeException(e);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("{} {} invoke finished instance id {}", componentTypeName, vertex.getName(),
- vertex.getInstanceID());
- }
-
- flushOutputs();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
deleted file mode 100755
index bc89241..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.streaming.api.collector.StreamOutput;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.io.BlockingQueueBroker;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class StreamIterationHead<OUT> extends StreamVertex<OUT, OUT> {
-
- private static final Logger LOG = LoggerFactory.getLogger(StreamIterationHead.class);
-
- private OutputHandler<OUT> outputHandler;
-
- private static int numSources;
- private Integer iterationId;
- @SuppressWarnings("rawtypes")
- private BlockingQueue<StreamRecord> dataChannel;
- private long iterationWaitTime;
- private boolean shouldWait;
-
- @SuppressWarnings("rawtypes")
- public StreamIterationHead() {
- numSources = newVertex();
- instanceID = numSources;
- dataChannel = new ArrayBlockingQueue<StreamRecord>(1);
- }
-
- @Override
- public void setInputsOutputs() {
- outputHandler = new OutputHandler<OUT>(this);
-
- iterationId = configuration.getIterationId();
- iterationWaitTime = configuration.getIterationWaitTime();
- shouldWait = iterationWaitTime > 0;
-
- try {
- BlockingQueueBroker.instance().handIn(iterationId.toString(), dataChannel);
- } catch (Exception e) {
-
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void invoke() throws Exception {
- if (LOG.isDebugEnabled()) {
- LOG.debug("SOURCE {} invoked with instance id {}", getName(), getInstanceID());
- }
-
- StreamRecord<OUT> nextRecord;
-
- List<StreamOutput<OUT>> outputs = new LinkedList<StreamOutput<OUT>>();
- for (StreamOutput<?> output : outputHandler.getOutputs()) {
- outputs.add((StreamOutput<OUT>) output);
- }
-
- while (true) {
- if (shouldWait) {
- nextRecord = dataChannel.poll(iterationWaitTime, TimeUnit.MILLISECONDS);
- } else {
- nextRecord = dataChannel.take();
- }
- if (nextRecord == null) {
- break;
- }
- for (StreamOutput<OUT> output : outputs) {
- output.collect(nextRecord.getObject());
- }
- }
-
- outputHandler.flushOutputs();
- }
-
- @Override
- protected void setInvokable() {
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
deleted file mode 100755
index b3ecdf8..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.io.BlockingQueueBroker;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class StreamIterationTail<IN> extends StreamVertex<IN, IN> {
-
- private static final Logger LOG = LoggerFactory.getLogger(StreamIterationTail.class);
-
- private InputHandler<IN> inputHandler;
-
- private Integer iterationId;
- @SuppressWarnings("rawtypes")
- private BlockingQueue<StreamRecord> dataChannel;
- private long iterationWaitTime;
- private boolean shouldWait;
-
- public StreamIterationTail() {
- }
-
- @Override
- public void setInputsOutputs() {
- try {
- inputHandler = new InputHandler<IN>(this);
-
- iterationId = configuration.getIterationId();
- iterationWaitTime = configuration.getIterationWaitTime();
- shouldWait = iterationWaitTime > 0;
- dataChannel = BlockingQueueBroker.instance().get(iterationId.toString());
- } catch (Exception e) {
- throw new StreamVertexException(String.format(
- "Cannot register inputs of StreamIterationSink %s", iterationId), e);
- }
- }
-
- @Override
- public void invoke() throws Exception {
- if (LOG.isDebugEnabled()) {
- LOG.debug("SINK {} invoked", getName());
- }
-
- forwardRecords();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("SINK {} invoke finished", getName());
- }
- }
-
- protected void forwardRecords() throws Exception {
- StreamRecord<IN> reuse = inputHandler.getInputSerializer().createInstance();
- while ((reuse = inputHandler.getInputIter().next(reuse)) != null) {
- if (!pushToQueue(reuse)) {
- break;
- }
- // TODO: Fix object reuse for iteration
- reuse = inputHandler.getInputSerializer().createInstance();
- }
- }
-
- private boolean pushToQueue(StreamRecord<IN> record) {
- try {
- if (shouldWait) {
- return dataChannel.offer(record, iterationWaitTime, TimeUnit.MILLISECONDS);
- } else {
- dataChannel.put(record);
- return true;
- }
- } catch (InterruptedException e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Pushing back record at iteration %s failed due to: {}", iterationId,
- StringUtils.stringifyException(e));
- }
- return false;
- }
- }
-
- @Override
- protected void setInvokable() {
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java
deleted file mode 100644
index 7fbab3b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-import org.apache.flink.streaming.api.StreamConfig;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.io.CoReaderIterator;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.MutableObjectIterator;
-
-public interface StreamTaskContext<OUT> {
-
- StreamConfig getConfig();
-
- ClassLoader getUserCodeClassLoader();
-
- <X> MutableObjectIterator<X> getInput(int index);
-
- <X> StreamRecordSerializer<X> getInputSerializer(int index);
-
- Collector<OUT> getOutputCollector();
-
- <X, Y> CoReaderIterator<X, Y> getCoReader();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
deleted file mode 100644
index 994b1fa..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-import java.util.Map;
-
-import org.apache.flink.runtime.execution.Environment;
-import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.streaming.api.StreamConfig;
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.io.CoReaderIterator;
-import org.apache.flink.streaming.state.OperatorState;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.MutableObjectIterator;
-
-public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTaskContext<OUT> {
-
- private static int numTasks;
-
- protected StreamConfig configuration;
- protected int instanceID;
- private static int numVertices = 0;
-
- private InputHandler<IN> inputHandler;
- protected OutputHandler<OUT> outputHandler;
- private StreamInvokable<IN, OUT> userInvokable;
-
- private StreamingRuntimeContext context;
- private Map<String, OperatorState<?>> states;
-
- protected ClassLoader userClassLoader;
-
- public StreamVertex() {
- userInvokable = null;
- numTasks = newVertex();
- instanceID = numTasks;
- }
-
- protected static int newVertex() {
- numVertices++;
- return numVertices;
- }
-
- @Override
- public void registerInputOutput() {
- initialize();
- setInputsOutputs();
- setInvokable();
- }
-
- protected void initialize() {
- this.userClassLoader = getUserCodeClassLoader();
- this.configuration = new StreamConfig(getTaskConfiguration());
- this.states = configuration.getOperatorStates(userClassLoader);
- this.context = createRuntimeContext(getEnvironment().getTaskName(), this.states);
- }
-
- protected <T> void invokeUserFunction(StreamInvokable<?, T> userInvokable) throws Exception {
- userInvokable.setRuntimeContext(context);
- userInvokable.open(getTaskConfiguration());
-
- for (ChainableInvokable<?, ?> invokable : outputHandler.chainedInvokables) {
- invokable.setRuntimeContext(context);
- invokable.open(getTaskConfiguration());
- }
-
- userInvokable.invoke();
- userInvokable.close();
-
- for (ChainableInvokable<?, ?> invokable : outputHandler.chainedInvokables) {
- invokable.close();
- }
-
- }
-
- public void setInputsOutputs() {
- inputHandler = new InputHandler<IN>(this);
- outputHandler = new OutputHandler<OUT>(this);
- }
-
- protected void setInvokable() {
- userInvokable = configuration.getUserInvokable(userClassLoader);
- userInvokable.setup(this);
- }
-
- public String getName() {
- return getEnvironment().getTaskName();
- }
-
- public int getInstanceID() {
- return instanceID;
- }
-
- public StreamingRuntimeContext createRuntimeContext(String taskName,
- Map<String, OperatorState<?>> states) {
- Environment env = getEnvironment();
- return new StreamingRuntimeContext(taskName, env, getUserCodeClassLoader(), states);
- }
-
- @Override
- public void invoke() throws Exception {
- outputHandler.invokeUserFunction("TASK", userInvokable);
- }
-
- @Override
- public StreamConfig getConfig() {
- return configuration;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <X> MutableObjectIterator<X> getInput(int index) {
- if (index == 0) {
- return (MutableObjectIterator<X>) inputHandler.getInputIter();
- } else {
- throw new IllegalArgumentException("There is only 1 input");
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <X> StreamRecordSerializer<X> getInputSerializer(int index) {
- if (index == 0) {
- return (StreamRecordSerializer<X>) inputHandler.getInputSerializer();
- } else {
- throw new IllegalArgumentException("There is only 1 input");
- }
- }
-
- @Override
- public Collector<OUT> getOutputCollector() {
- return outputHandler.getCollector();
- }
-
- @Override
- public <X, Y> CoReaderIterator<X, Y> getCoReader() {
- throw new IllegalArgumentException("CoReader not available");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertexException.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertexException.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertexException.java
deleted file mode 100644
index ed8b91e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertexException.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-/**
- * An exception that is thrown by the stream verices when encountering an
- * illegal condition.
- */
-public class StreamVertexException extends RuntimeException {
-
- /**
- * Serial version UID for serialization interoperability.
- */
- private static final long serialVersionUID = 8392043527067472439L;
-
- /**
- * Creates a compiler exception with no message and no cause.
- */
- public StreamVertexException() {
- }
-
- /**
- * Creates a compiler exception with the given message and no cause.
- *
- * @param message
- * The message for the exception.
- */
- public StreamVertexException(String message) {
- super(message);
- }
-
- /**
- * Creates a compiler exception with the given cause and no message.
- *
- * @param cause
- * The <tt>Throwable</tt> that caused this exception.
- */
- public StreamVertexException(Throwable cause) {
- super(cause);
- }
-
- /**
- * Creates a compiler exception with the given message and cause.
- *
- * @param message
- * The message for the exception.
- * @param cause
- * The <tt>Throwable</tt> that caused this exception.
- */
- public StreamVertexException(String message, Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamingRuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamingRuntimeContext.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamingRuntimeContext.java
deleted file mode 100644
index a1a64e2..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamingRuntimeContext.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.streamvertex;
-
-import java.util.Map;
-
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.common.functions.util.RuntimeUDFContext;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.execution.Environment;
-import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
-import org.apache.flink.runtime.operators.util.TaskConfig;
-import org.apache.flink.streaming.state.OperatorState;
-
-/**
- * Implementation of the {@link RuntimeContext}, created by runtime stream UDF
- * operators.
- */
-public class StreamingRuntimeContext extends RuntimeUDFContext {
-
- public Environment env;
- private final Map<String, OperatorState<?>> operatorStates;
-
- public StreamingRuntimeContext(String name, Environment env, ClassLoader userCodeClassLoader,
- Map<String, OperatorState<?>> operatorStates) {
- super(name, env.getNumberOfSubtasks(), env.getIndexInSubtaskGroup(),
- userCodeClassLoader, env.getCopyTask());
- this.env = env;
- this.operatorStates = operatorStates;
- }
-
- /**
- * Returns the operator state registered by the given name for the operator.
- *
- * @param name
- * Name of the operator state to be returned.
- * @return The operator state.
- */
- public OperatorState<?> getState(String name) {
- if (operatorStates == null) {
- throw new RuntimeException("No state has been registered for this operator.");
- } else {
- OperatorState<?> state = operatorStates.get(name);
- if (state != null) {
- return state;
- } else {
- throw new RuntimeException("No state has been registered for the name: " + name);
- }
- }
-
- }
-
- /**
- * Returns the input split provider associated with the operator.
- *
- * @return The input split provider.
- */
- public InputSplitProvider getInputSplitProvider() {
- return env.getInputSplitProvider();
- }
-
- /**
- * Returns the stub parameters associated with the {@link TaskConfig} of the
- * operator.
- *
- * @return The stub parameters.
- */
- public Configuration getTaskStubParameters() {
- return new TaskConfig(env.getTaskConfiguration()).getStubParameters();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java
deleted file mode 100644
index 15aaf51..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.deltafunction;
-
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
-
-/**
- * This delta function calculates the cosine distance between two given vectors.
- * The cosine distance is defined as: cosineDistance=1-cosineSimilarity
- *
- * Cosine similarity: http://en.wikipedia.org/wiki/Cosine_similarity
- *
- * @param <DATA>
- * The input data type. This delta function works with a double[],
- * but can extract/convert to it from any other given object in case
- * the respective extractor has been set. See
- * {@link ExtractionAwareDeltaFunction} for more information.
- */
-public class CosineDistance<DATA> extends ExtractionAwareDeltaFunction<DATA, double[]> {
-
- /**
- * auto-generated id
- */
- private static final long serialVersionUID = -1217813582965151599L;
-
- public CosineDistance() {
- super(null);
- }
-
- public CosineDistance(Extractor<DATA, double[]> converter) {
- super(converter);
- }
-
- @Override
- public double getNestedDelta(double[] oldDataPoint, double[] newDataPoint) {
- if (isNullvector(oldDataPoint, newDataPoint)) {
- return 0;
- }
-
- double sum1 = 0;
- double sum2 = 0;
- for (int i = 0; i < oldDataPoint.length; i++) {
- sum1 += oldDataPoint[i] * oldDataPoint[i];
- sum2 += newDataPoint[i] * newDataPoint[i];
- }
- sum1 = Math.sqrt(sum1);
- sum2 = Math.sqrt(sum2);
-
- return 1d - (dotProduct(oldDataPoint, newDataPoint) / (sum1 * sum2));
- }
-
- private double dotProduct(double[] a, double[] b) {
- double result = 0;
- for (int i = 0; i < a.length; i++) {
- result += a[i] * b[i];
- }
- return result;
- }
-
- private boolean isNullvector(double[]... vectors) {
- outer: for (double[] v : vectors) {
- for (double field : v) {
- if (field != 0) {
- continue outer;
- }
- }
- // This position is only reached in case all fields are 0.
- return true;
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java
deleted file mode 100644
index b2223d6..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.deltafunction;
-
-import java.io.Serializable;
-
-/**
- * This interface allows the implementation of a function which calculates the
- * delta between two data points. Delta functions might be used in delta
- * policies and allow flexible adaptive windowing based on the arriving data
- * points.
- *
- * @param <DATA>
- * The type of input data which can be compared using this function.
- */
-public interface DeltaFunction<DATA> extends Serializable {
-
- /**
- * Calculates the delta between two given data points.
- *
- * @param oldDataPoint
- * the old data point.
- * @param newDataPoint
- * the new data point.
- * @return the delta between the two given points.
- */
- public double getDelta(DATA oldDataPoint, DATA newDataPoint);
-
-}
[02/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
new file mode 100644
index 0000000..177a9ee
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream,
+ SingleOutputStreamOperator, GroupedDataStream}
+import scala.reflect.ClassTag
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.functions.MapFunction
+import org.apache.flink.streaming.api.invokable.operator.MapInvokable
+import org.apache.flink.util.Collector
+import org.apache.flink.api.common.functions.FlatMapFunction
+import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable
+import org.apache.flink.api.common.functions.ReduceFunction
+import org.apache.flink.streaming.api.invokable.StreamInvokable
+import org.apache.flink.streaming.api.invokable.operator.{ GroupedReduceInvokable, StreamReduceInvokable }
+import org.apache.flink.api.common.functions.ReduceFunction
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.common.functions.FilterFunction
+import org.apache.flink.streaming.api.function.sink.SinkFunction
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
+import org.apache.flink.streaming.api.windowing.policy.{ EvictionPolicy, TriggerPolicy }
+import org.apache.flink.streaming.api.collector.OutputSelector
+import scala.collection.JavaConversions._
+import java.util.HashMap
+import org.apache.flink.streaming.api.function.aggregation.SumFunction
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy
+
+class DataStream[T](javaStream: JavaStream[T]) {
+
+ /**
+ * Gets the underlying java DataStream object.
+ */
+ def getJavaStream: JavaStream[T] = javaStream
+
+ /**
+ * Sets the degree of parallelism of this operation. This must be greater than 1.
+ */
+ def setParallelism(dop: Int): DataStream[T] = {
+ javaStream match {
+ case ds: SingleOutputStreamOperator[_, _] => ds.setParallelism(dop)
+ case _ =>
+ throw new UnsupportedOperationException("Operator " + javaStream.toString + " cannot " +
+ "have " +
+ "parallelism.")
+ }
+ this
+ }
+
+ /**
+ * Returns the degree of parallelism of this operation.
+ */
+ def getParallelism: Int = javaStream match {
+ case op: SingleOutputStreamOperator[_, _] => op.getParallelism
+ case _ =>
+ throw new UnsupportedOperationException("Operator " + javaStream.toString + " does not have" +
+ " " +
+ "parallelism.")
+ }
+
+ def setChainingStrategy(strategy: ChainingStrategy): DataStream[T] = {
+ javaStream match {
+ case ds: SingleOutputStreamOperator[_, _] => ds.setChainingStrategy(strategy)
+ case _ =>
+ throw new UnsupportedOperationException("Only supported for operators.")
+ }
+ this
+ }
+
+ /**
+ * Creates a new DataStream by merging DataStream outputs of
+ * the same type with each other. The DataStreams merged using this operator
+ * will be transformed simultaneously.
+ *
+ */
+ def merge(dataStreams: DataStream[T]*): DataStream[T] =
+ javaStream.merge(dataStreams.map(_.getJavaStream): _*)
+
+ /**
+ * Creates a new ConnectedDataStream by connecting
+ * DataStream outputs of different type with each other. The
+ * DataStreams connected using this operators can be used with CoFunctions.
+ *
+ */
+ def connect[T2](dataStream: DataStream[T2]): ConnectedDataStream[T, T2] =
+ javaStream.connect(dataStream.getJavaStream)
+
+ /**
+ * Groups the elements of a DataStream by the given key positions (for tuple/array types) to
+ * be used with grouped operators like grouped reduce or grouped aggregations
+ *
+ */
+ def groupBy(fields: Int*): DataStream[T] = javaStream.groupBy(fields: _*)
+
+ /**
+ * Groups the elements of a DataStream by the given field expressions to
+ * be used with grouped operators like grouped reduce or grouped aggregations
+ *
+ */
+ def groupBy(firstField: String, otherFields: String*): DataStream[T] =
+ javaStream.groupBy(firstField +: otherFields.toArray: _*)
+
+ /**
+ * Groups the elements of a DataStream by the given K key to
+ * be used with grouped operators like grouped reduce or grouped aggregations
+ *
+ */
+ def groupBy[K: TypeInformation](fun: T => K): DataStream[T] = {
+
+ val keyExtractor = new KeySelector[T, K] {
+ val cleanFun = clean(fun)
+ def getKey(in: T) = cleanFun(in)
+ }
+ javaStream.groupBy(keyExtractor)
+ }
+
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are broadcasted to every parallel instance of the next component. This
+ * setting only effects the how the outputs will be distributed between the
+ * parallel instances of the next processing operator.
+ *
+ */
+ def broadcast: DataStream[T] = javaStream.broadcast()
+
+ /**
+ * Sets the partitioning of the DataStream so that the output values all go to
+ * the first instance of the next processing operator. Use this setting with care
+ * since it might cause a serious performance bottleneck in the application.
+ */
+ def global: DataStream[T] = javaStream.global()
+
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are shuffled to the next component. This setting only effects the how the
+ * outputs will be distributed between the parallel instances of the next
+ * processing operator.
+ *
+ */
+ def shuffle: DataStream[T] = javaStream.shuffle()
+
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are forwarded to the local subtask of the next component (whenever
+ * possible). This is the default partitioner setting. This setting only
+ * effects the how the outputs will be distributed between the parallel
+ * instances of the next processing operator.
+ *
+ */
+ def forward: DataStream[T] = javaStream.forward()
+
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are distributed evenly to the next component.This setting only effects
+ * the how the outputs will be distributed between the parallel instances of
+ * the next processing operator.
+ *
+ */
+ def distribute: DataStream[T] = javaStream.distribute()
+
+ /**
+ * Initiates an iterative part of the program that creates a loop by feeding
+ * back data streams. To create a streaming iteration the user needs to define
+ * a transformation that creates two DataStreams.The first one one is the output
+ * that will be fed back to the start of the iteration and the second is the output
+ * stream of the iterative part.
+ * <p>
+ * stepfunction: initialStream => (feedback, output)
+ * <p>
+ * A common pattern is to use output splitting to create feedback and output DataStream.
+ * Please refer to the .split(...) method of the DataStream
+ * <p>
+ * By default a DataStream with iteration will never terminate, but the user
+ * can use the maxWaitTime parameter to set a max waiting time for the iteration head.
+ * If no data received in the set time the stream terminates.
+ *
+ *
+ */
+ def iterate[R](stepFunction: DataStream[T] => (DataStream[T], DataStream[R]),
+ maxWaitTimeMillis:Long = 0): DataStream[R] = {
+ val iterativeStream = javaStream.iterate(maxWaitTimeMillis)
+
+ val (feedback, output) = stepFunction(new DataStream[T](iterativeStream))
+ iterativeStream.closeWith(feedback.getJavaStream)
+ output
+ }
+
+ /**
+ * Applies an aggregation that that gives the current maximum of the data stream at
+ * the given position.
+ *
+ */
+ def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
+
+ /**
+ * Applies an aggregation that that gives the current maximum of the data stream at
+ * the given field.
+ *
+ */
+ def max(field: String): DataStream[T] = aggregate(AggregationType.MAX, field)
+
+ /**
+ * Applies an aggregation that that gives the current minimum of the data stream at
+ * the given position.
+ *
+ */
+ def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
+
+ /**
+ * Applies an aggregation that that gives the current minimum of the data stream at
+ * the given field.
+ *
+ */
+ def min(field: String): DataStream[T] = aggregate(AggregationType.MIN, field)
+
+ /**
+ * Applies an aggregation that sums the data stream at the given position.
+ *
+ */
+ def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
+
+ /**
+ * Applies an aggregation that sums the data stream at the given field.
+ *
+ */
+ def sum(field: String): DataStream[T] = aggregate(AggregationType.SUM, field)
+
+ /**
+ * Applies an aggregation that that gives the current minimum element of the data stream by
+ * the given position. When equality, the first element is returned with the minimal value.
+ *
+ */
+ def minBy(position: Int): DataStream[T] = aggregate(AggregationType
+ .MINBY, position)
+
+ /**
+ * Applies an aggregation that that gives the current minimum element of the data stream by
+ * the given field. When equality, the first element is returned with the minimal value.
+ *
+ */
+ def minBy(field: String): DataStream[T] = aggregate(AggregationType
+ .MINBY, field )
+
+ /**
+ * Applies an aggregation that that gives the current maximum element of the data stream by
+ * the given position. When equality, the first element is returned with the maximal value.
+ *
+ */
+ def maxBy(position: Int): DataStream[T] =
+ aggregate(AggregationType.MAXBY, position)
+
+ /**
+ * Applies an aggregation that that gives the current maximum element of the data stream by
+ * the given field. When equality, the first element is returned with the maximal value.
+ *
+ */
+ def maxBy(field: String): DataStream[T] =
+ aggregate(AggregationType.MAXBY, field)
+
+ private def aggregate(aggregationType: AggregationType, field: String): DataStream[T] = {
+ val position = fieldNames2Indices(javaStream.getType(), Array(field))(0)
+ aggregate(aggregationType, position)
+ }
+
+ private def aggregate(aggregationType: AggregationType, position: Int):
+ DataStream[T] = {
+
+ val jStream = javaStream.asInstanceOf[JavaStream[Product]]
+ val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
+
+ val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
+
+ val reducer = aggregationType match {
+ case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(outType.getTypeAt(position).
+ getTypeClass()))
+ case _ => new agg.ProductComparableAggregator(aggregationType, true)
+ }
+
+ val invokable = jStream match {
+ case groupedStream: GroupedDataStream[_] => new GroupedReduceInvokable(reducer,
+ groupedStream.getKeySelector())
+ case _ => new StreamReduceInvokable(reducer)
+ }
+ new DataStream[Product](jStream.transform("aggregation", jStream.getType(),
+ invokable)).asInstanceOf[DataStream[T]]
+ }
+
+ /**
+ * Creates a new DataStream containing the current number (count) of
+ * received records.
+ *
+ */
+ def count: DataStream[Long] = new DataStream[java.lang.Long](
+ javaStream.count()).asInstanceOf[DataStream[Long]]
+
+ /**
+ * Creates a new DataStream by applying the given function to every element of this DataStream.
+ */
+ def map[R: TypeInformation: ClassTag](fun: T => R): DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+ val mapper = new MapFunction[T, R] {
+ val cleanFun = clean(fun)
+ def map(in: T): R = cleanFun(in)
+ }
+
+ javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper))
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element of this DataStream.
+ */
+ def map[R: TypeInformation: ClassTag](mapper: MapFunction[T, R]): DataStream[R] = {
+ if (mapper == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+
+ javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper))
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element and flattening
+ * the results.
+ */
+ def flatMap[R: TypeInformation: ClassTag](flatMapper: FlatMapFunction[T, R]): DataStream[R] = {
+ if (flatMapper == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ javaStream.transform("flatMap", implicitly[TypeInformation[R]],
+ new FlatMapInvokable[T, R](flatMapper))
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element and flattening
+ * the results.
+ */
+ def flatMap[R: TypeInformation: ClassTag](fun: (T, Collector[R]) => Unit): DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ val flatMapper = new FlatMapFunction[T, R] {
+ val cleanFun = clean(fun)
+ def flatMap(in: T, out: Collector[R]) { cleanFun(in, out) }
+ }
+ flatMap(flatMapper)
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element and flattening
+ * the results.
+ */
+ def flatMap[R: TypeInformation: ClassTag](fun: T => TraversableOnce[R]): DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ val flatMapper = new FlatMapFunction[T, R] {
+ val cleanFun = clean(fun)
+ def flatMap(in: T, out: Collector[R]) { cleanFun(in) foreach out.collect }
+ }
+ flatMap(flatMapper)
+ }
+
+ /**
+ * Creates a new [[DataStream]] by reducing the elements of this DataStream
+ * using an associative reduce function.
+ */
+ def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
+ if (reducer == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ javaStream match {
+ case ds: GroupedDataStream[_] => javaStream.transform("reduce",
+ javaStream.getType(), new GroupedReduceInvokable[T](reducer, ds.getKeySelector()))
+ case _ => javaStream.transform("reduce", javaStream.getType(),
+ new StreamReduceInvokable[T](reducer))
+ }
+ }
+
+ /**
+ * Creates a new [[DataStream]] by reducing the elements of this DataStream
+ * using an associative reduce function.
+ */
+ def reduce(fun: (T, T) => T): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ val reducer = new ReduceFunction[T] {
+ val cleanFun = clean(fun)
+ def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
+ }
+ reduce(reducer)
+ }
+
+ /**
+ * Creates a new DataStream that contains only the elements satisfying the given filter predicate.
+ */
+ def filter(filter: FilterFunction[T]): DataStream[T] = {
+ if (filter == null) {
+ throw new NullPointerException("Filter function must not be null.")
+ }
+ javaStream.filter(filter)
+ }
+
+ /**
+ * Creates a new DataStream that contains only the elements satisfying the given filter predicate.
+ */
+ def filter(fun: T => Boolean): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Filter function must not be null.")
+ }
+ val filter = new FilterFunction[T] {
+ val cleanFun = clean(fun)
+ def filter(in: T) = cleanFun(in)
+ }
+ this.filter(filter)
+ }
+
+ /**
+ * Create a WindowedDataStream that can be used to apply
+ * transformation like .reduce(...) or aggregations on
+ * preset chunks(windows) of the data stream. To define the windows one or
+ * more WindowingHelper-s such as Time, Count and
+ * Delta can be used.</br></br> When applied to a grouped data
+ * stream, the windows (evictions) and slide sizes (triggers) will be
+ * computed on a per group basis. </br></br> For more advanced control over
+ * the trigger and eviction policies please use to
+ * window(List(triggers), List(evicters))
+ */
+ def window(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
+ javaStream.window(windowingHelper: _*)
+
+ /**
+ * Create a WindowedDataStream using the given TriggerPolicy-s and EvictionPolicy-s.
+ * Windowing can be used to apply transformation like .reduce(...) or aggregations on
+ * preset chunks(windows) of the data stream.</br></br>For most common
+ * use-cases please refer to window(WindowingHelper[_]*)
+ *
+ */
+ def window(triggers: List[TriggerPolicy[T]], evicters: List[EvictionPolicy[T]]):
+ WindowedDataStream[T] = javaStream.window(triggers, evicters)
+
+ /**
+ *
+ * Operator used for directing tuples to specific named outputs using an
+ * OutputSelector. Calling this method on an operator creates a new
+ * SplitDataStream.
+ */
+ def split(selector: OutputSelector[T]): SplitDataStream[T] = javaStream.split(selector)
+
+ /**
+ * Creates a new SplitDataStream that contains only the elements satisfying the
+ * given output selector predicate.
+ */
+ def split(fun: T => String): SplitDataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("OutputSelector must not be null.")
+ }
+ val selector = new OutputSelector[T] {
+ val cleanFun = clean(fun)
+ def select(in: T): java.lang.Iterable[String] = {
+ List(cleanFun(in))
+ }
+ }
+ split(selector)
+ }
+
+ /**
+ * Initiates a temporal Join transformation that joins the elements of two
+ * data streams on key equality over a specified time window.
+ *
+ * This method returns a StreamJoinOperator on which the
+ * .onWindow(..) should be called to define the
+ * window, and then the .where(..) and .equalTo(..) methods can be used to defin
+ * the join keys.</p> The user can also use the apply method of the returned JoinedStream
+ * to use custom join function.
+ *
+ */
+ def join[R](stream: DataStream[R]): StreamJoinOperator[T, R] =
+ new StreamJoinOperator[T, R](javaStream, stream.getJavaStream)
+
+ /**
+ * Initiates a temporal cross transformation that builds all pair
+ * combinations of elements of both DataStreams, i.e., it builds a Cartesian
+ * product.
+ *
+ * This method returns a StreamJoinOperator on which the
+ * .onWindow(..) should be called to define the
+ * window, and then the .where(..) and .equalTo(..) methods can be used to defin
+ * the join keys.</p> The user can also use the apply method of the returned JoinedStream
+ * to use custom join function.
+ *
+ */
+ def cross[R](stream: DataStream[R]): StreamCrossOperator[T, R] =
+ new StreamCrossOperator[T, R](javaStream, stream.getJavaStream)
+
+ /**
+ * Writes a DataStream to the standard output stream (stdout). For each
+ * element of the DataStream the result of .toString is
+ * written.
+ *
+ */
+ def print(): DataStream[T] = javaStream.print()
+
+ /**
+ * Writes a DataStream to the file specified by path in text format. The
+ * writing is performed periodically, in every millis milliseconds. For
+ * every element of the DataStream the result of .toString
+ * is written.
+ *
+ */
+ def writeAsText(path: String, millis: Long = 0): DataStream[T] =
+ javaStream.writeAsText(path, millis)
+
+ /**
+ * Writes a DataStream to the file specified by path in text format. The
+ * writing is performed periodically, in every millis milliseconds. For
+ * every element of the DataStream the result of .toString
+ * is written.
+ *
+ */
+ def writeAsCsv(path: String, millis: Long = 0): DataStream[T] =
+ javaStream.writeAsCsv(path, millis)
+
+ /**
+ * Adds the given sink to this DataStream. Only streams with sinks added
+ * will be executed once the StreamExecutionEnvironment.execute(...)
+ * method is called.
+ *
+ */
+ def addSink(sinkFuntion: SinkFunction[T]): DataStream[T] =
+ javaStream.addSink(sinkFuntion)
+
+ /**
+ * Adds the given sink to this DataStream. Only streams with sinks added
+ * will be executed once the StreamExecutionEnvironment.execute(...)
+ * method is called.
+ *
+ */
+ def addSink(fun: T => Unit): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Sink function must not be null.")
+ }
+ val sinkFunction = new SinkFunction[T] {
+ val cleanFun = clean(fun)
+ def invoke(in: T) = cleanFun(in)
+ }
+ this.addSink(sinkFunction)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
new file mode 100644
index 0000000..9e33f80
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
+
+/**
+ * The SplitDataStream represents an operator that has been split using an
+ * {@link OutputSelector}. Named outputs can be selected using the
+ * {@link #select} function. To apply a transformation on the whole output simply call
+ * the appropriate method on this stream.
+ *
+ * @param <OUT>
+ * The type of the output.
+ */
+class SplitDataStream[T](javaStream: SplitJavaStream[T]) extends DataStream[T](javaStream){
+
+ /**
+ * Sets the output names for which the next operator will receive values.
+ */
+ def select(outputNames: String*): DataStream[T] = javaStream.select(outputNames: _*)
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
new file mode 100644
index 0000000..a408ec0
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import scala.reflect.ClassTag
+import org.apache.commons.lang.Validate
+import org.apache.flink.api.common.functions.CrossFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.scala.typeutils.CaseClassSerializer
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream}
+import org.apache.flink.streaming.api.function.co.CrossWindowFunction
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
+import java.util.concurrent.TimeUnit
+
+class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
+ TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
+
+ override def createNextWindowOperator(): StreamCrossOperator.CrossWindow[I1, I2] = {
+
+ val crossWindowFunction = StreamCrossOperator.getCrossWindowFunction(this,
+ (l: I1, r: I2) => (l, r))
+
+ val returnType = new CaseClassTypeInfo[(I1, I2)](
+
+ classOf[(I1, I2)], Seq(input1.getType, input2.getType), Array("_1", "_2")) {
+
+ override def createSerializer: TypeSerializer[(I1, I2)] = {
+ val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](getArity)
+ for (i <- 0 until getArity) {
+ fieldSerializers(i) = types(i).createSerializer
+ }
+
+ new CaseClassSerializer[(I1, I2)](classOf[(I1, I2)], fieldSerializers) {
+ override def createInstance(fields: Array[AnyRef]) = {
+ (fields(0).asInstanceOf[I1], fields(1).asInstanceOf[I2])
+ }
+ }
+ }
+ }
+
+ val javaStream = input1.connect(input2).addGeneralWindowCombine(
+ crossWindowFunction,
+ returnType, windowSize,
+ slideInterval, timeStamp1, timeStamp2)
+
+ new StreamCrossOperator.CrossWindow[I1, I2](this, javaStream)
+ }
+}
+object StreamCrossOperator {
+
+ private[flink] class CrossWindow[I1, I2](op: StreamCrossOperator[I1, I2],
+ javaStream: JavaStream[(I1, I2)]) extends
+ DataStream[(I1, I2)](javaStream) with TemporalWindow[CrossWindow[I1, I2]] {
+
+ /**
+ * Sets a wrapper for the crossed elements. For each crossed pair, the result of the udf
+ * call will be emitted.
+ *
+ */
+ def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
+
+ val invokable = new CoWindowInvokable[I1, I2, R](
+ clean(getCrossWindowFunction(op, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
+ op.timeStamp2)
+
+ javaStream.getExecutionEnvironment().getStreamGraph().setInvokable(javaStream.getId(),
+ invokable)
+
+ javaStream.setType(implicitly[TypeInformation[R]])
+ }
+
+ override def every(length: Long, timeUnit: TimeUnit): CrossWindow[I1, I2] = {
+ every(timeUnit.toMillis(length))
+ }
+
+ override def every(length: Long): CrossWindow[I1, I2] = {
+ val builder = javaStream.getExecutionEnvironment().getStreamGraph()
+ val invokable = builder.getInvokable(javaStream.getId())
+ invokable.asInstanceOf[CoWindowInvokable[_,_,_]].setSlideSize(length)
+ this
+ }
+ }
+
+ private[flink] def getCrossWindowFunction[I1, I2, R](op: StreamCrossOperator[I1, I2],
+ crossFunction: (I1, I2) => R):
+ CrossWindowFunction[I1, I2, R] = {
+ Validate.notNull(crossFunction, "Join function must not be null.")
+
+ val crossFun = new CrossFunction[I1, I2, R] {
+ val cleanFun = op.input1.clean(crossFunction)
+
+ override def cross(first: I1, second: I2): R = {
+ cleanFun(first, second)
+ }
+ }
+
+ new CrossWindowFunction[I1, I2, R](crossFun)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
new file mode 100644
index 0000000..394673c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import scala.reflect.ClassTag
+import org.apache.commons.lang.Validate
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JavaEnv}
+import org.apache.flink.streaming.api.function.source.{ FromElementsFunction, SourceFunction }
+import org.apache.flink.util.Collector
+import org.apache.flink.api.scala.ClosureCleaner
+import org.apache.flink.streaming.api.function.source.FileMonitoringFunction.WatchType
+
+class StreamExecutionEnvironment(javaEnv: JavaEnv) {
+
+ /**
+ * Sets the degree of parallelism (DOP) for operations executed through this environment.
+ * Setting a DOP of x here will cause all operators (such as join, map, reduce) to run with
+ * x parallel instances. This value can be overridden by specific operations using
+ * [[DataStream.setParallelism]].
+ */
+ def setDegreeOfParallelism(degreeOfParallelism: Int): Unit = {
+ javaEnv.setDegreeOfParallelism(degreeOfParallelism)
+ }
+
+ /**
+ * Returns the default degree of parallelism for this execution environment. Note that this
+ * value can be overridden by individual operations using [[DataStream.setParallelism]]
+ */
+ def getDegreeOfParallelism = javaEnv.getDegreeOfParallelism
+
+ /**
+ * Sets the maximum time frequency (milliseconds) for the flushing of the
+ * output buffers. By default the output buffers flush frequently to provide
+ * low latency and to aid smooth developer experience. Setting the parameter
+ * can result in three logical modes:
+ *
+ * <ul>
+ * <li>
+ * A positive integer triggers flushing periodically by that integer</li>
+ * <li>
+ * 0 triggers flushing after every record thus minimizing latency</li>
+ * <li>
+ * -1 triggers flushing only when the output buffer is full thus maximizing
+ * throughput</li>
+ * </ul>
+ *
+ */
+ def setBufferTimeout(timeoutMillis: Long): StreamExecutionEnvironment = {
+ javaEnv.setBufferTimeout(timeoutMillis)
+ this
+ }
+
+ /**
+ * Gets the default buffer timeout set for this environment
+ */
+ def getBufferTimout: Long = javaEnv.getBufferTimeout()
+
+ /**
+ * Creates a DataStream that represents the Strings produced by reading the
+ * given file line wise. The file will be read with the system's default
+ * character set.
+ *
+ */
+ def readTextFile(filePath: String): DataStream[String] =
+ javaEnv.readTextFile(filePath)
+
+ /**
+ * Creates a DataStream that contains the contents of file created while
+ * system watches the given path. The file will be read with the system's
+ * default character set. The user can check the monitoring interval in milliseconds,
+ * and the way file modifications are handled. By default it checks for only new files
+ * every 100 milliseconds.
+ *
+ */
+ def readFileStream(StreamPath: String, intervalMillis: Long = 100, watchType: WatchType =
+ WatchType.ONLY_NEW_FILES): DataStream[String] =
+ javaEnv.readFileStream(StreamPath, intervalMillis, watchType)
+
+ /**
+ * Creates a new DataStream that contains the strings received infinitely
+ * from socket. Received strings are decoded by the system's default
+ * character set.
+ *
+ */
+ def socketTextStream(hostname: String, port: Int, delimiter: Char): DataStream[String] =
+ javaEnv.socketTextStream(hostname, port, delimiter)
+
+ /**
+ * Creates a new DataStream that contains the strings received infinitely
+ * from socket. Received strings are decoded by the system's default
+ * character set, uses '\n' as delimiter.
+ *
+ */
+ def socketTextStream(hostname: String, port: Int): DataStream[String] =
+ javaEnv.socketTextStream(hostname, port)
+
+ /**
+ * Creates a new DataStream that contains a sequence of numbers.
+ *
+ */
+ def generateSequence(from: Long, to: Long): DataStream[Long] = {
+ new DataStream[java.lang.Long](javaEnv.generateSequence(from, to)).
+ asInstanceOf[DataStream[Long]]
+ }
+
+ /**
+ * Creates a DataStream that contains the given elements. The elements must all be of the
+ * same type and must be serializable.
+ *
+ * * Note that this operation will result in a non-parallel data source, i.e. a data source with
+ * a degree of parallelism of one.
+ */
+ def fromElements[T: ClassTag: TypeInformation](data: T*): DataStream[T] = {
+ val typeInfo = implicitly[TypeInformation[T]]
+ fromCollection(data)(implicitly[ClassTag[T]], typeInfo)
+ }
+
+ /**
+ * Creates a DataStream from the given non-empty [[Seq]]. The elements need to be serializable
+ * because the framework may move the elements into the cluster if needed.
+ *
+ * Note that this operation will result in a non-parallel data source, i.e. a data source with
+ * a degree of parallelism of one.
+ */
+ def fromCollection[T: ClassTag: TypeInformation](
+ data: Seq[T]): DataStream[T] = {
+ Validate.notNull(data, "Data must not be null.")
+ val typeInfo = implicitly[TypeInformation[T]]
+
+ val sourceFunction = new FromElementsFunction[T](scala.collection.JavaConversions
+ .asJavaCollection(data))
+
+ javaEnv.addSource(sourceFunction, typeInfo)
+ }
+
+ /**
+ * Create a DataStream using a user defined source function for arbitrary
+ * source functionality. By default sources have a parallelism of 1.
+ * To enable parallel execution, the user defined source should implement
+ * ParallelSourceFunction or extend RichParallelSourceFunction.
+ * In these cases the resulting source will have the parallelism of the environment.
+ * To change this afterwards call DataStreamSource.setParallelism(int)
+ *
+ */
+ def addSource[T: ClassTag: TypeInformation](function: SourceFunction[T]): DataStream[T] = {
+ Validate.notNull(function, "Function must not be null.")
+ val cleanFun = StreamExecutionEnvironment.clean(function)
+ val typeInfo = implicitly[TypeInformation[T]]
+ javaEnv.addSource(cleanFun, typeInfo)
+ }
+
+ /**
+ * Create a DataStream using a user defined source function for arbitrary
+ * source functionality.
+ *
+ */
+ def addSource[T: ClassTag: TypeInformation](function: Collector[T] => Unit): DataStream[T] = {
+ Validate.notNull(function, "Function must not be null.")
+ val sourceFunction = new SourceFunction[T] {
+ val cleanFun = StreamExecutionEnvironment.clean(function)
+ override def invoke(out: Collector[T]) {
+ cleanFun(out)
+ }
+ }
+ addSource(sourceFunction)
+ }
+
+ /**
+ * Triggers the program execution. The environment will execute all parts of
+ * the program that have resulted in a "sink" operation. Sink operations are
+ * for example printing results or forwarding them to a message queue.
+ * <p>
+ * The program execution will be logged and displayed with a generated
+ * default name.
+ *
+ */
+ def execute() = javaEnv.execute()
+
+ /**
+ * Triggers the program execution. The environment will execute all parts of
+ * the program that have resulted in a "sink" operation. Sink operations are
+ * for example printing results or forwarding them to a message queue.
+ * <p>
+ * The program execution will be logged and displayed with the provided name
+ *
+ */
+ def execute(jobName: String) = javaEnv.execute(jobName)
+
+ /**
+ * Creates the plan with which the system will execute the program, and
+ * returns it as a String using a JSON representation of the execution data
+ * flow graph. Note that this needs to be called, before the plan is
+ * executed.
+ *
+ */
+ def getExecutionPlan() = javaEnv.getStreamGraph.getStreamingPlanAsJSON
+
+}
+
+object StreamExecutionEnvironment {
+
+ private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
+ ClosureCleaner.clean(f, checkSerializable)
+ f
+ }
+
+ /**
+ * Creates an execution environment that represents the context in which the program is
+ * currently executed. If the program is invoked standalone, this method returns a local
+ * execution environment. If the program is invoked from within the command line client
+ * to be submitted to a cluster, this method returns the execution environment of this cluster.
+ */
+ def getExecutionEnvironment: StreamExecutionEnvironment = {
+ new StreamExecutionEnvironment(JavaEnv.getExecutionEnvironment)
+ }
+
+ /**
+ * Creates a local execution environment. The local execution environment will run the program in
+ * a multi-threaded fashion in the same JVM as the environment was created in. The default degree
+ * of parallelism of the local environment is the number of hardware contexts (CPU cores/threads).
+ */
+ def createLocalEnvironment(
+ degreeOfParallelism: Int = Runtime.getRuntime.availableProcessors()):
+ StreamExecutionEnvironment = {
+ new StreamExecutionEnvironment(JavaEnv.createLocalEnvironment(degreeOfParallelism))
+ }
+
+ /**
+ * Creates a remote execution environment. The remote environment sends (parts of) the program to
+ * a cluster for execution. Note that all file paths used in the program must be accessible from
+ * the cluster. The execution will use the cluster's default degree of parallelism, unless the
+ * parallelism is set explicitly via [[StreamExecutionEnvironment.setDegreeOfParallelism()]].
+ *
+ * @param host The host name or address of the master (JobManager),
+ * where the program should be executed.
+ * @param port The port of the master (JobManager), where the program should be executed.
+ * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the
+ * program uses
+ * user-defined functions, user-defined input formats, or any libraries,
+ * those must be
+ * provided in the JAR files.
+ */
+ def createRemoteEnvironment(host: String, port: Int, jarFiles: String*):
+ StreamExecutionEnvironment = {
+ new StreamExecutionEnvironment(JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*))
+ }
+
+ /**
+ * Creates a remote execution environment. The remote environment sends (parts of) the program
+ * to a cluster for execution. Note that all file paths used in the program must be accessible
+ * from the cluster. The execution will use the specified degree of parallelism.
+ *
+ * @param host The host name or address of the master (JobManager),
+ * where the program should be executed.
+ * @param port The port of the master (JobManager), where the program should be executed.
+ * @param degreeOfParallelism The degree of parallelism to use during the execution.
+ * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the
+ * program uses
+ * user-defined functions, user-defined input formats, or any libraries,
+ * those must be
+ * provided in the JAR files.
+ */
+ def createRemoteEnvironment(
+ host: String,
+ port: Int,
+ degreeOfParallelism: Int,
+ jarFiles: String*): StreamExecutionEnvironment = {
+ val javaEnv = JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*)
+ javaEnv.setDegreeOfParallelism(degreeOfParallelism)
+ new StreamExecutionEnvironment(javaEnv)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
new file mode 100644
index 0000000..1bd1bfb
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import scala.Array.canBuildFrom
+import scala.reflect.ClassTag
+import org.apache.commons.lang.Validate
+import org.apache.flink.api.common.functions.JoinFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.java.operators.Keys
+import org.apache.flink.api.scala.typeutils.CaseClassSerializer
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.function.co.JoinWindowFunction
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.util.keys.KeySelectorUtil
+import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
+import java.util.concurrent.TimeUnit
+
+class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
+TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
+
+ override def createNextWindowOperator() = {
+ new StreamJoinOperator.JoinWindow[I1, I2](this)
+ }
+}
+
+object StreamJoinOperator {
+
+ class JoinWindow[I1, I2](private[flink]op: StreamJoinOperator[I1, I2]) extends
+ TemporalWindow[JoinWindow[I1, I2]] {
+
+ private[flink] val type1 = op.input1.getType()
+
+ /**
+ * Continues a temporal Join transformation by defining
+ * the fields in the first stream to be used as keys for the join.
+ * The resulting incomplete join can be completed by JoinPredicate.equalTo()
+ * to define the second key.
+ */
+ def where(fields: Int*) = {
+ new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(fields.toArray, type1), type1))
+ }
+
+ /**
+ * Continues a temporal Join transformation by defining
+ * the fields in the first stream to be used as keys for the join.
+ * The resulting incomplete join can be completed by JoinPredicate.equalTo()
+ * to define the second key.
+ */
+ def where(firstField: String, otherFields: String*) =
+ new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(firstField +: otherFields.toArray, type1), type1))
+
+ /**
+ * Continues a temporal Join transformation by defining
+ * the keyselector function that will be used to extract keys from the first stream
+ * for the join.
+ * The resulting incomplete join can be completed by JoinPredicate.equalTo()
+ * to define the second key.
+ */
+ def where[K: TypeInformation](fun: (I1) => K) = {
+ val keyType = implicitly[TypeInformation[K]]
+ val keyExtractor = new KeySelector[I1, K] {
+ val cleanFun = op.input1.clean(fun)
+ def getKey(in: I1) = cleanFun(in)
+ }
+ new JoinPredicate[I1, I2](op, keyExtractor)
+ }
+
+ override def every(length: Long, timeUnit: TimeUnit): JoinWindow[I1, I2] = {
+ every(timeUnit.toMillis(length))
+ }
+
+ override def every(length: Long): JoinWindow[I1, I2] = {
+ op.slideInterval = length
+ this
+ }
+
+ }
+
+ class JoinPredicate[I1, I2](private[flink] val op: StreamJoinOperator[I1, I2],
+ private[flink] val keys1: KeySelector[I1, _]) {
+ private[flink] var keys2: KeySelector[I2, _] = null
+ private[flink] val type2 = op.input2.getType()
+
+ /**
+ * Creates a temporal join transformation by defining the second join key.
+ * The returned transformation wrapes each joined element pair in a tuple2:
+ * (first, second)
+ * To define a custom wrapping, use JoinedStream.apply(...)
+ */
+ def equalTo(fields: Int*): JoinedStream[I1, I2] = {
+ finish(KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(fields.toArray, type2), type2))
+ }
+
+ /**
+ * Creates a temporal join transformation by defining the second join key.
+ * The returned transformation wrapes each joined element pair in a tuple2:
+ * (first, second)
+ * To define a custom wrapping, use JoinedStream.apply(...)
+ */
+ def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] =
+ finish(KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(firstField +: otherFields.toArray, type2), type2))
+
+ /**
+ * Creates a temporal join transformation by defining the second join key.
+ * The returned transformation wrapes each joined element pair in a tuple2:
+ * (first, second)
+ * To define a custom wrapping, use JoinedStream.apply(...)
+ */
+ def equalTo[K: TypeInformation](fun: (I2) => K): JoinedStream[I1, I2] = {
+ val keyType = implicitly[TypeInformation[K]]
+ val keyExtractor = new KeySelector[I2, K] {
+ val cleanFun = op.input1.clean(fun)
+ def getKey(in: I2) = cleanFun(in)
+ }
+ finish(keyExtractor)
+ }
+
+ private def finish(keys2: KeySelector[I2, _]): JoinedStream[I1, I2] = {
+ this.keys2 = keys2
+ new JoinedStream[I1, I2](this, createJoinOperator())
+ }
+
+ private def createJoinOperator(): JavaStream[(I1, I2)] = {
+
+ val returnType = new CaseClassTypeInfo[(I1, I2)](
+
+ classOf[(I1, I2)], Seq(op.input1.getType, op.input2.getType), Array("_1", "_2")) {
+
+ override def createSerializer: TypeSerializer[(I1, I2)] = {
+ val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](getArity)
+ for (i <- 0 until getArity) {
+ fieldSerializers(i) = types(i).createSerializer
+ }
+
+ new CaseClassSerializer[(I1, I2)](classOf[(I1, I2)], fieldSerializers) {
+ override def createInstance(fields: Array[AnyRef]) = {
+ (fields(0).asInstanceOf[I1], fields(1).asInstanceOf[I2])
+ }
+ }
+ }
+ }
+
+ return op.input1.groupBy(keys1).connect(op.input2.groupBy(keys2))
+ .addGeneralWindowCombine(getJoinWindowFunction(this, (_, _)),
+ returnType, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
+ }
+ }
+
+ class JoinedStream[I1, I2](jp: JoinPredicate[I1, I2], javaStream: JavaStream[(I1, I2)]) extends
+ DataStream[(I1, I2)](javaStream) {
+
+ private val op = jp.op
+
+ /**
+ * Sets a wrapper for the joined elements. For each joined pair, the result of the
+ * udf call will be emitted.
+ */
+ def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
+
+ val invokable = new CoWindowInvokable[I1, I2, R](
+ clean(getJoinWindowFunction(jp, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
+ op.timeStamp2)
+
+ javaStream.getExecutionEnvironment().getStreamGraph().setInvokable(javaStream.getId(),
+ invokable)
+
+ javaStream.setType(implicitly[TypeInformation[R]])
+ }
+ }
+
+ private[flink] def getJoinWindowFunction[I1, I2, R](jp: JoinPredicate[I1, I2],
+ joinFunction: (I1, I2) => R) = {
+ Validate.notNull(joinFunction, "Join function must not be null.")
+
+ val joinFun = new JoinFunction[I1, I2, R] {
+
+ val cleanFun = jp.op.input1.clean(joinFunction)
+
+ override def join(first: I1, second: I2): R = {
+ cleanFun(first, second)
+ }
+ }
+
+ new JoinWindowFunction[I1, I2, R](jp.keys1, jp.keys2, joinFun)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala
new file mode 100644
index 0000000..fd3a4a9
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import org.apache.flink.streaming.api.datastream.temporaloperator.{ TemporalOperator => JTempOp }
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
+import org.apache.flink.streaming.api.windowing.helper.Timestamp
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment._
+
+abstract class TemporalOperator[I1, I2, OP <: TemporalWindow[OP]](
+ i1: JavaStream[I1], i2: JavaStream[I2]) extends JTempOp[I1, I2, OP](i1, i2) {
+
+ def onWindow(length: Long, ts1: I1 => Long, ts2: I2 => Long, startTime: Long = 0): OP = {
+ val timeStamp1 = getTS(ts1)
+ val timeStamp2 = getTS(ts2)
+ onWindow(length, timeStamp1, timeStamp2, startTime)
+ }
+
+ def getTS[R](ts: R => Long): Timestamp[R] = {
+ new Timestamp[R] {
+ val cleanFun = clean(ts, true)
+ def getTimestamp(in: R) = cleanFun(in)
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
new file mode 100644
index 0000000..5c734bf
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import scala.Array.canBuildFrom
+import scala.collection.JavaConversions.iterableAsScalaIterable
+import scala.reflect.ClassTag
+
+import org.apache.flink.api.common.functions.GroupReduceFunction
+import org.apache.flink.api.common.functions.ReduceFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
+import org.apache.flink.streaming.api.datastream.{WindowedDataStream => JavaWStream}
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
+import org.apache.flink.streaming.api.function.aggregation.SumFunction
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
+import org.apache.flink.streaming.api.windowing.helper._
+import org.apache.flink.util.Collector
+
+class WindowedDataStream[T](javaStream: JavaWStream[T]) {
+
+ /**
+ * Defines the slide size (trigger frequency) for the windowed data stream.
+ * This controls how often the user defined function will be triggered on
+ * the window.
+ */
+ def every(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
+ javaStream.every(windowingHelper: _*)
+
+ /**
+ * Groups the elements of the WindowedDataStream using the given
+ * field positions. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * DataStream.window(...) operator on an already grouped data stream.
+ *
+ */
+ def groupBy(fields: Int*): WindowedDataStream[T] = javaStream.groupBy(fields: _*)
+
+ /**
+ * Groups the elements of the WindowedDataStream using the given
+ * field expressions. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * DataStream.window(...) operator on an already grouped data stream.
+ *
+ */
+ def groupBy(firstField: String, otherFields: String*): WindowedDataStream[T] =
+ javaStream.groupBy(firstField +: otherFields.toArray: _*)
+
+ /**
+ * Groups the elements of the WindowedDataStream using the given
+ * KeySelector function. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * DataStream.window(...) operator on an already grouped data stream.
+ *
+ */
+ def groupBy[K: TypeInformation](fun: T => K): WindowedDataStream[T] = {
+
+ val keyExtractor = new KeySelector[T, K] {
+ val cleanFun = clean(fun)
+ def getKey(in: T) = cleanFun(in)
+ }
+ javaStream.groupBy(keyExtractor)
+ }
+
+ /**
+ * Applies a reduce transformation on the windowed data stream by reducing
+ * the current window at every trigger.
+ *
+ */
+ def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
+ if (reducer == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ javaStream.reduce(reducer)
+ }
+
+ /**
+ * Applies a reduce transformation on the windowed data stream by reducing
+ * the current window at every trigger.
+ *
+ */
+ def reduce(fun: (T, T) => T): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ val reducer = new ReduceFunction[T] {
+ val cleanFun = clean(fun)
+ def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
+ }
+ reduce(reducer)
+ }
+
+ /**
+ * Applies a reduceGroup transformation on the windowed data stream by reducing
+ * the current window at every trigger. In contrast with the simple binary reduce operator,
+ * groupReduce exposes the whole window through the Iterable interface.
+ * </br>
+ * </br>
+ * Whenever possible try to use reduce instead of groupReduce for increased efficiency
+ */
+ def reduceGroup[R: ClassTag: TypeInformation](reducer: GroupReduceFunction[T, R]):
+ DataStream[R] = {
+ if (reducer == null) {
+ throw new NullPointerException("GroupReduce function must not be null.")
+ }
+ javaStream.reduceGroup(reducer, implicitly[TypeInformation[R]])
+ }
+
+ /**
+ * Applies a reduceGroup transformation on the windowed data stream by reducing
+ * the current window at every trigger. In contrast with the simple binary reduce operator,
+ * groupReduce exposes the whole window through the Iterable interface.
+ * </br>
+ * </br>
+ * Whenever possible try to use reduce instead of groupReduce for increased efficiency
+ */
+ def reduceGroup[R: ClassTag: TypeInformation](fun: (Iterable[T], Collector[R]) => Unit):
+ DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("GroupReduce function must not be null.")
+ }
+ val reducer = new GroupReduceFunction[T, R] {
+ val cleanFun = clean(fun)
+ def reduce(in: java.lang.Iterable[T], out: Collector[R]) = { cleanFun(in, out) }
+ }
+ reduceGroup(reducer)
+ }
+
+ /**
+ * Applies an aggregation that that gives the maximum of the elements in the window at
+ * the given position.
+ *
+ */
+ def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
+
+ /**
+ * Applies an aggregation that that gives the maximum of the elements in the window at
+ * the given field.
+ *
+ */
+ def max(field: String): DataStream[T] = aggregate(AggregationType.MAX, field)
+
+ /**
+ * Applies an aggregation that that gives the minimum of the elements in the window at
+ * the given position.
+ *
+ */
+ def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
+
+ /**
+ * Applies an aggregation that that gives the minimum of the elements in the window at
+ * the given field.
+ *
+ */
+ def min(field: String): DataStream[T] = aggregate(AggregationType.MIN, field)
+
+ /**
+ * Applies an aggregation that sums the elements in the window at the given position.
+ *
+ */
+ def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
+
+ /**
+ * Applies an aggregation that sums the elements in the window at the given field.
+ *
+ */
+ def sum(field: String): DataStream[T] = aggregate(AggregationType.SUM, field)
+
+ /**
+ * Applies an aggregation that that gives the maximum element of the window by
+ * the given position. When equality, returns the first.
+ *
+ */
+ def maxBy(position: Int): DataStream[T] = aggregate(AggregationType.MAXBY,
+ position)
+
+ /**
+ * Applies an aggregation that that gives the maximum element of the window by
+ * the given field. When equality, returns the first.
+ *
+ */
+ def maxBy(field: String): DataStream[T] = aggregate(AggregationType.MAXBY,
+ field)
+
+ /**
+ * Applies an aggregation that that gives the minimum element of the window by
+ * the given position. When equality, returns the first.
+ *
+ */
+ def minBy(position: Int): DataStream[T] = aggregate(AggregationType.MINBY,
+ position)
+
+ /**
+ * Applies an aggregation that that gives the minimum element of the window by
+ * the given field. When equality, returns the first.
+ *
+ */
+ def minBy(field: String): DataStream[T] = aggregate(AggregationType.MINBY,
+ field)
+
+ private def aggregate(aggregationType: AggregationType, field: String): DataStream[T] = {
+ val position = fieldNames2Indices(javaStream.getType(), Array(field))(0)
+ aggregate(aggregationType, position)
+ }
+
+ def aggregate(aggregationType: AggregationType, position: Int):
+ DataStream[T] = {
+
+ val jStream = javaStream.asInstanceOf[JavaWStream[Product]]
+ val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
+
+ val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
+
+ val reducer = aggregationType match {
+ case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(
+ outType.getTypeAt(position).getTypeClass()))
+ case _ => new agg.ProductComparableAggregator(aggregationType, true)
+ }
+
+ new DataStream[Product](jStream.reduce(reducer)).asInstanceOf[DataStream[T]]
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
new file mode 100644
index 0000000..222eb6d
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api
+
+import _root_.scala.reflect.ClassTag
+import language.experimental.macros
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.scala.typeutils.{CaseClassTypeInfo, TypeUtils}
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
+import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
+import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaConStream }
+
+package object scala {
+ // We have this here so that we always have generated TypeInformationS when
+ // using the Scala API
+ implicit def createTypeInformation[T]: TypeInformation[T] = macro TypeUtils.createTypeInfo[T]
+
+ implicit def javaToScalaStream[R](javaStream: JavaStream[R]): DataStream[R] =
+ new DataStream[R](javaStream)
+
+ implicit def javaToScalaWindowedStream[R](javaWStream: JavaWStream[R]): WindowedDataStream[R] =
+ new WindowedDataStream[R](javaWStream)
+
+ implicit def javaToScalaSplitStream[R](javaStream: SplitJavaStream[R]): SplitDataStream[R] =
+ new SplitDataStream[R](javaStream)
+
+ implicit def javaToScalaConnectedStream[IN1, IN2](javaStream: JavaConStream[IN1, IN2]):
+ ConnectedDataStream[IN1, IN2] = new ConnectedDataStream[IN1, IN2](javaStream)
+
+ implicit def seqToFlinkSource[T: ClassTag: TypeInformation](scalaSeq: Seq[T]) : DataStream[T] =
+ StreamExecutionEnvironment.getExecutionEnvironment.fromCollection(scalaSeq)
+
+
+ private[flink] def fieldNames2Indices(
+ typeInfo: TypeInformation[_],
+ fields: Array[String]): Array[Int] = {
+ typeInfo match {
+ case ti: CaseClassTypeInfo[_] =>
+ val result = ti.getFieldIndices(fields)
+
+ if (result.contains(-1)) {
+ throw new IllegalArgumentException("Fields '" + fields.mkString(", ") +
+ "' are not valid for '" + ti.toString + "'.")
+ }
+
+ result
+
+ case _ =>
+ throw new UnsupportedOperationException("Specifying fields by name is only" +
+ "supported on Case Classes (for now).")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala
new file mode 100644
index 0000000..eedee0e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala.windowing
+
+import org.apache.flink.streaming.api.windowing.helper.{ Delta => JavaDelta }
+import org.apache.commons.lang.Validate
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction
+
+object Delta {
+
+ /**
+ * Creates a delta helper representing a delta trigger or eviction policy.
+ * </br></br> This policy calculates a delta between the data point which
+ * triggered last and the currently arrived data point. It triggers if the
+ * delta is higher than a specified threshold. </br></br> In case it gets
+ * used for eviction, this policy starts from the first element of the
+ * buffer and removes all elements from the buffer which have a higher delta
+ * then the threshold. As soon as there is an element with a lower delta,
+ * the eviction stops.
+ */
+ def of[T](threshold: Double, deltaFunction: (T, T) => Double, initVal: T): JavaDelta[T] = {
+ Validate.notNull(deltaFunction, "Delta function must not be null")
+ val df = new DeltaFunction[T] {
+ val cleanFun = clean(deltaFunction)
+ override def getDelta(first: T, second: T) = cleanFun(first, second)
+ }
+ JavaDelta.of(threshold, df, initVal)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
new file mode 100644
index 0000000..9a69369
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala.windowing
+
+import java.util.concurrent.TimeUnit
+import org.apache.flink.streaming.api.windowing.helper.{ Time => JavaTime }
+
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.windowing.helper.Timestamp
+import org.apache.commons.lang.Validate
+
+object Time {
+
+ /**
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using System time.
+ *
+ */
+ def of(windowSize: Long, timeUnit: TimeUnit): JavaTime[_] =
+ JavaTime.of(windowSize, timeUnit)
+
+ /**
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using a user defined timestamp extractor.
+ *
+ */
+ def of[R](windowSize: Long, timestamp: R => Long, startTime: Long = 0): JavaTime[R] = {
+ Validate.notNull(timestamp, "Timestamp must not be null.")
+ val ts = new Timestamp[R] {
+ val fun = clean(timestamp, true)
+ override def getTimestamp(in: R) = fun(in)
+ }
+ JavaTime.of(windowSize, ts, startTime)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/pom.xml b/flink-staging/flink-streaming/pom.xml
new file mode 100644
index 0000000..43b8181
--- /dev/null
+++ b/flink-staging/flink-streaming/pom.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-staging</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-streaming-parent</artifactId>
+ <name>flink-streaming</name>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>flink-streaming-core</module>
+ <module>flink-streaming-scala</module>
+ <module>flink-streaming-examples</module>
+ <module>flink-streaming-connectors</module>
+ </modules>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-compiler</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-tachyon/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-tachyon/pom.xml b/flink-staging/flink-tachyon/pom.xml
new file mode 100644
index 0000000..65aa295
--- /dev/null
+++ b/flink-staging/flink-tachyon/pom.xml
@@ -0,0 +1,117 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-staging</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-tachyon</artifactId>
+ <name>flink-tachyon</name>
+
+ <packaging>jar</packaging>
+
+ <!--
+ This is a Hadoop2 only flink module.
+ -->
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java-examples</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-avro</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.tachyonproject</groupId>
+ <artifactId>tachyon</artifactId>
+ <version>0.5.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.tachyonproject</groupId>
+ <artifactId>tachyon</artifactId>
+ <version>0.5.0</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ <version>7.6.8.v20121106</version><!--$NO-MVN-MAN-VER$-->
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ <version>${hadoop.version}</version><!--$NO-MVN-MAN-VER$-->
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ <version>${hadoop.version}</version><!--$NO-MVN-MAN-VER$-->
+ </dependency>
+ </dependencies>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>7.6.8.v20121106</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>7.6.8.v20121106</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-tachyon/src/test/java/org/apache/flink/tachyon/HDFSTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-tachyon/src/test/java/org/apache/flink/tachyon/HDFSTest.java b/flink-staging/flink-tachyon/src/test/java/org/apache/flink/tachyon/HDFSTest.java
new file mode 100644
index 0000000..7318894
--- /dev/null
+++ b/flink-staging/flink-tachyon/src/test/java/org/apache/flink/tachyon/HDFSTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tachyon;
+
+
+import org.apache.commons.io.IOUtils;
+import org.apache.flink.api.common.io.FileOutputFormat;
+import org.apache.flink.api.java.io.AvroOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.examples.java.wordcount.WordCount;
+import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringWriter;
+
+/**
+ * This test should logically be located in the 'flink-runtime' tests. However, this project
+ * has already all dependencies required (flink-java-examples). Also, the DOPOneExecEnv is here.
+ */
+public class HDFSTest {
+
+ private String hdfsURI;
+ private MiniDFSCluster hdfsCluster;
+ private org.apache.hadoop.fs.Path hdPath;
+ private org.apache.hadoop.fs.FileSystem hdfs;
+
+ @Before
+ public void createHDFS() {
+ try {
+ Configuration hdConf = new Configuration();
+
+ File baseDir = new File("./target/hdfs/hdfsTest").getAbsoluteFile();
+ FileUtil.fullyDelete(baseDir);
+ hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
+ MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
+ hdfsCluster = builder.build();
+
+ hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() +"/";
+
+ hdPath = new org.apache.hadoop.fs.Path("/test");
+ hdfs = hdPath.getFileSystem(hdConf);
+ FSDataOutputStream stream = hdfs.create(hdPath);
+ for(int i = 0; i < 10; i++) {
+ stream.write("Hello HDFS\n".getBytes());
+ }
+ stream.close();
+
+ } catch(Throwable e) {
+ e.printStackTrace();
+ Assert.fail("Test failed " + e.getMessage());
+ }
+ }
+
+ @After
+ public void destroyHDFS() {
+ try {
+ hdfs.delete(hdPath, false);
+ hdfsCluster.shutdown();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ @Test
+ public void testHDFS() {
+
+ Path file = new Path(hdfsURI + hdPath);
+ org.apache.hadoop.fs.Path result = new org.apache.hadoop.fs.Path(hdfsURI + "/result");
+ try {
+ FileSystem fs = file.getFileSystem();
+ Assert.assertTrue("Must be HadoopFileSystem", fs instanceof HadoopFileSystem);
+ new TachyonFileSystemWrapperTest.DopOneTestEnvironment();
+ try {
+ WordCount.main(new String[]{file.toString(), result.toString()});
+ } catch(Throwable t) {
+ t.printStackTrace();
+ Assert.fail("Test failed with " + t.getMessage());
+ }
+ Assert.assertTrue("No result file present", hdfs.exists(result));
+ // validate output:
+ org.apache.hadoop.fs.FSDataInputStream inStream = hdfs.open(result);
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(inStream, writer);
+ String resultString = writer.toString();
+
+ Assert.assertEquals("hdfs 10\n" +
+ "hello 10\n", resultString);
+ inStream.close();
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail("Error in test: " + e.getMessage() );
+ }
+ }
+
+ @Test
+ public void testAvroOut() {
+ String type = "one";
+ AvroOutputFormat<String> avroOut =
+ new AvroOutputFormat<String>( String.class );
+
+ org.apache.hadoop.fs.Path result = new org.apache.hadoop.fs.Path(hdfsURI + "/avroTest");
+
+ avroOut.setOutputFilePath(new Path(result.toString()));
+ avroOut.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
+ avroOut.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);
+
+ try {
+ avroOut.open(0, 2);
+ avroOut.writeRecord(type);
+ avroOut.close();
+
+ avroOut.open(1, 2);
+ avroOut.writeRecord(type);
+ avroOut.close();
+
+
+ Assert.assertTrue("No result file present", hdfs.exists(result));
+ FileStatus[] files = hdfs.listStatus(result);
+ Assert.assertEquals(2, files.length);
+ for(FileStatus file : files) {
+ Assert.assertTrue("1.avro".equals(file.getPath().getName()) || "2.avro".equals(file.getPath().getName()));
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+}
[44/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessagingFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessagingFunction.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessagingFunction.java
deleted file mode 100644
index 794dce5..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/MessagingFunction.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java.record;
-
-import java.io.Serializable;
-import java.util.Iterator;
-
-import org.apache.flink.api.common.aggregators.Aggregator;
-import org.apache.flink.api.common.functions.IterationRuntimeContext;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.types.Key;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.Value;
-import org.apache.flink.util.Collector;
-
-public abstract class MessagingFunction<VertexKey extends Key<VertexKey>, VertexValue extends Value, Message extends Value, EdgeValue extends Value> implements Serializable {
-
- // --------------------------------------------------------------------------------------------
- // Public API Methods
- // --------------------------------------------------------------------------------------------
-
- public abstract void sendMessages(VertexKey vertexKey, VertexValue vertexValue) throws Exception;
-
- public void setup(Configuration config) throws Exception {}
-
- public void preSuperstep() throws Exception {}
-
- public void postSuperstep() throws Exception {}
-
-
- public Iterator<Edge<VertexKey, EdgeValue>> getOutgoingEdges() {
- if (edgesUsed) {
- throw new IllegalStateException("Can use either 'getOutgoingEdges()' or 'sendMessageToAllTargets()'.");
- }
-
- edgesUsed = true;
- edgeIter.set(edges);
- return edgeIter;
- }
-
- public void sendMessageToAllNeighbors(Message m) {
- if (edgesUsed) {
- throw new IllegalStateException("Can use either 'getOutgoingEdges()' or 'sendMessageToAllTargets()'.");
- }
-
- edgesUsed = true;
- while (edges.hasNext()) {
- Record next = edges.next();
- VertexKey k = next.getField(1, this.keyClass);
- outValue.setField(0, k);
- outValue.setField(1, m);
- out.collect(outValue);
- }
- }
-
- public void sendMessageTo(VertexKey target, Message m) {
- outValue.setField(0, target);
- outValue.setField(1, m);
- out.collect(outValue);
- }
-
- // --------------------------------------------------------------------------------------------
-
- public int getSuperstep() {
- return this.runtimeContext.getSuperstepNumber();
- }
-
- public <T extends Aggregator<?>> T getIterationAggregator(String name) {
- return this.runtimeContext.<T>getIterationAggregator(name);
- }
-
- public <T extends Value> T getPreviousIterationAggregate(String name) {
- return this.runtimeContext.<T>getPreviousIterationAggregate(name);
- }
-
- // --------------------------------------------------------------------------------------------
- // internal methods and state
- // --------------------------------------------------------------------------------------------
-
- private Record outValue;
-
- private IterationRuntimeContext runtimeContext;
-
- private Iterator<Record> edges;
-
- private Collector<Record> out;
-
- private EdgesIterator<VertexKey, EdgeValue> edgeIter;
-
- private Class<VertexKey> keyClass;
-
- private boolean edgesUsed;
-
-
- @SuppressWarnings("unchecked")
- void init(IterationRuntimeContext context, VertexKey keyHolder, EdgeValue edgeValueHolder) {
- this.runtimeContext = context;
- this.edgeIter = new EdgesIterator<VertexKey, EdgeValue>(keyHolder, edgeValueHolder);
- this.outValue = new Record();
- this.keyClass = (Class<VertexKey>) keyHolder.getClass();
- }
-
- void set(Iterator<Record> edges, Collector<Record> out) {
- this.edges = edges;
- this.out = out;
- this.edgesUsed = false;
- }
-
- private static final long serialVersionUID = 1L;
-
- private static final class EdgesIterator<VertexKey extends Key<VertexKey>, EdgeValue extends Value> implements Iterator<Edge<VertexKey, EdgeValue>> {
-
- private Iterator<Record> input;
- private VertexKey keyHolder;
- private EdgeValue edgeValueHolder;
-
- private Edge<VertexKey, EdgeValue> edge = new Edge<VertexKey, EdgeValue>();
-
- EdgesIterator(VertexKey keyHolder, EdgeValue edgeValueHolder) {
- this.keyHolder = keyHolder;
- this.edgeValueHolder = edgeValueHolder;
- }
-
- void set(Iterator<Record> input) {
- this.input = input;
- }
-
- @Override
- public boolean hasNext() {
- return input.hasNext();
- }
-
- @Override
- public Edge<VertexKey, EdgeValue> next() {
- Record next = input.next();
- next.getFieldInto(0, keyHolder);
- next.getFieldInto(1, edgeValueHolder);
- edge.set(keyHolder, edgeValueHolder);
- return edge;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java
deleted file mode 100644
index f647e5d..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java.record;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.flink.api.common.aggregators.AggregatorRegistry;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.java.record.functions.CoGroupFunction;
-import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFieldsFirst;
-import org.apache.flink.api.java.record.operators.CoGroupOperator;
-import org.apache.flink.api.java.record.operators.DeltaIteration;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.types.Key;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.Value;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.util.ReflectionUtil;
-
-@SuppressWarnings("deprecation")
-public class SpargelIteration {
-
- private static final String DEFAULT_NAME = "<unnamed vertex-centric iteration>";
-
- private final DeltaIteration iteration;
-
- private final Class<? extends Key<?>> vertexKey;
- private final Class<? extends Value> vertexValue;
- private final Class<? extends Value> messageType;
- private final Class<? extends Value> edgeValue;
-
- private final CoGroupOperator vertexUpdater;
- private final CoGroupOperator messager;
-
-
- // ----------------------------------------------------------------------------------
-
- public <VertexKey extends Key<VertexKey>, VertexValue extends Value, Message extends Value, EdgeValue extends Value>
- SpargelIteration(MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf,
- VertexUpdateFunction<VertexKey, VertexValue, Message> uf)
- {
- this(mf, uf, DEFAULT_NAME);
- }
-
- public <VertexKey extends Key<VertexKey>, VertexValue extends Value, Message extends Value, EdgeValue extends Value> SpargelIteration(
- MessagingFunction<VertexKey, VertexValue, Message, EdgeValue> mf, VertexUpdateFunction<VertexKey, VertexValue, Message> uf,
- String name)
- {
- // get the types
- this.vertexKey = ReflectionUtil.getTemplateType1(mf.getClass());
- this.vertexValue = ReflectionUtil.getTemplateType2(mf.getClass());
- this.messageType = ReflectionUtil.getTemplateType3(mf.getClass());
- this.edgeValue = ReflectionUtil.getTemplateType4(mf.getClass());
-
- if (vertexKey == null || vertexValue == null || messageType == null || edgeValue == null) {
- throw new RuntimeException();
- }
-
- // instantiate the data flow
- this.iteration = new DeltaIteration(0, name);
-
- this.messager = CoGroupOperator.builder(MessagingDriver.class, vertexKey, 0, 0)
- .input2(iteration.getWorkset())
- .name("Message Sender")
- .build();
- this.vertexUpdater = CoGroupOperator.builder(VertexUpdateDriver.class, vertexKey, 0, 0)
- .input1(messager)
- .input2(iteration.getSolutionSet())
- .name("Vertex Updater")
- .build();
-
- iteration.setNextWorkset(vertexUpdater);
- iteration.setSolutionSetDelta(vertexUpdater);
-
- // parameterize the data flow
- try {
- Configuration vertexUdfParams = vertexUpdater.getParameters();
- InstantiationUtil.writeObjectToConfig(uf, vertexUdfParams, VertexUpdateDriver.UDF_PARAM);
- vertexUdfParams.setClass(VertexUpdateDriver.KEY_PARAM, vertexKey);
- vertexUdfParams.setClass(VertexUpdateDriver.VALUE_PARAM, vertexValue);
- vertexUdfParams.setClass(VertexUpdateDriver.MESSAGE_PARAM, messageType);
-
- Configuration messageUdfParams = messager.getParameters();
- InstantiationUtil.writeObjectToConfig(mf, messageUdfParams, MessagingDriver.UDF_PARAM);
- messageUdfParams.setClass(MessagingDriver.KEY_PARAM, vertexKey);
- messageUdfParams.setClass(MessagingDriver.VALUE_PARAM, vertexValue);
- messageUdfParams.setClass(MessagingDriver.MESSAGE_PARAM, messageType);
- messageUdfParams.setClass(MessagingDriver.EDGE_PARAM, edgeValue);
- }
- catch (IOException e) {
- throw new RuntimeException("Could not serialize the UDFs for distribution" +
- (e.getMessage() == null ? '.' : ": " + e.getMessage()), e);
- }
- }
-
- // ----------------------------------------------------------------------------------
- // inputs and outputs
- // ----------------------------------------------------------------------------------
-
- public void setVertexInput(Operator<Record> c) {
- this.iteration.setInitialSolutionSet(c);
- this.iteration.setInitialWorkset(c);
- }
-
- public void setEdgesInput(Operator<Record> c) {
- this.messager.setFirstInput(c);
- }
-
- public Operator<?> getOutput() {
- return this.iteration;
- }
-
- public void setDegreeOfParallelism(int dop) {
- this.iteration.setDegreeOfParallelism(dop);
- }
-
- public void setNumberOfIterations(int iterations) {
- this.iteration.setMaximumNumberOfIterations(iterations);
- }
-
- public AggregatorRegistry getAggregators() {
- return this.iteration.getAggregators();
- }
-
- // --------------------------------------------------------------------------------------------
- // Wrapping UDFs
- // --------------------------------------------------------------------------------------------
-
- @ConstantFieldsFirst(0)
- public static final class VertexUpdateDriver<K extends Key<K>, V extends Value, M extends Value> extends CoGroupFunction {
-
- private static final long serialVersionUID = 1L;
-
- private static final String UDF_PARAM = "spargel.udf";
- private static final String KEY_PARAM = "spargel.key-type";
- private static final String VALUE_PARAM = "spargel.value-type";
- private static final String MESSAGE_PARAM = "spargel.message-type";
-
- private VertexUpdateFunction<K, V, M> vertexUpdateFunction;
-
- private K vertexKey;
- private V vertexValue;
- private MessageIterator<M> messageIter;
-
- @Override
- public void coGroup(Iterator<Record> messages, Iterator<Record> vertex, Collector<Record> out) throws Exception {
-
- if (vertex.hasNext()) {
- Record first = vertex.next();
- first.getFieldInto(0, vertexKey);
- first.getFieldInto(1, vertexValue);
- messageIter.setSource(messages);
- vertexUpdateFunction.setOutput(first, out);
- vertexUpdateFunction.updateVertex(vertexKey, vertexValue, messageIter);
- } else {
- if (messages.hasNext()) {
- String message = "Target vertex does not exist!.";
- try {
- Record next = messages.next();
- next.getFieldInto(0, vertexKey);
- message = "Target vertex '" + vertexKey + "' does not exist!.";
- } catch (Throwable t) {}
- throw new Exception(message);
- } else {
- throw new Exception();
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void open(Configuration parameters) throws Exception {
- // instantiate only the first time
- if (vertexUpdateFunction == null) {
- ClassLoader cl = getRuntimeContext().getUserCodeClassLoader();
-
- Class<K> vertexKeyClass = parameters.getClass(KEY_PARAM, null, cl);
- Class<V> vertexValueClass = parameters.getClass(VALUE_PARAM, null, cl);
- Class<M> messageClass = parameters.getClass(MESSAGE_PARAM, null, cl);
-
- vertexKey = InstantiationUtil.instantiate(vertexKeyClass, Key.class);
- vertexValue = InstantiationUtil.instantiate(vertexValueClass, Value.class);
- messageIter = new MessageIterator<M>(InstantiationUtil.instantiate(messageClass, Value.class));
-
- ClassLoader ucl = getRuntimeContext().getUserCodeClassLoader();
-
- try {
- this.vertexUpdateFunction = (VertexUpdateFunction<K, V, M>) InstantiationUtil.readObjectFromConfig(parameters, UDF_PARAM, ucl);
- } catch (Exception e) {
- String message = e.getMessage() == null ? "." : ": " + e.getMessage();
- throw new Exception("Could not instantiate VertexUpdateFunction" + message, e);
- }
-
- this.vertexUpdateFunction.init(getIterationRuntimeContext());
- this.vertexUpdateFunction.setup(parameters);
- }
- this.vertexUpdateFunction.preSuperstep();
- }
-
- @Override
- public void close() throws Exception {
- this.vertexUpdateFunction.postSuperstep();
- }
- }
-
- public static final class MessagingDriver<K extends Key<K>, V extends Value, M extends Value, E extends Value> extends CoGroupFunction {
-
- private static final long serialVersionUID = 1L;
-
- private static final String UDF_PARAM = "spargel.udf";
- private static final String KEY_PARAM = "spargel.key-type";
- private static final String VALUE_PARAM = "spargel.value-type";
- private static final String MESSAGE_PARAM = "spargel.message-type";
- private static final String EDGE_PARAM = "spargel.edge-value";
-
-
- private MessagingFunction<K, V, M, E> messagingFunction;
-
- private K vertexKey;
- private V vertexValue;
-
- @Override
- public void coGroup(Iterator<Record> edges, Iterator<Record> state, Collector<Record> out) throws Exception {
- if (state.hasNext()) {
- Record first = state.next();
- first.getFieldInto(0, vertexKey);
- first.getFieldInto(1, vertexValue);
- messagingFunction.set(edges, out);
- messagingFunction.sendMessages(vertexKey, vertexValue);
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void open(Configuration parameters) throws Exception {
- // instantiate only the first time
- if (messagingFunction == null) {
- ClassLoader cl = getRuntimeContext().getUserCodeClassLoader();
-
- Class<K> vertexKeyClass = parameters.getClass(KEY_PARAM, null, cl);
- Class<V> vertexValueClass = parameters.getClass(VALUE_PARAM, null, cl);
-// Class<M> messageClass = parameters.getClass(MESSAGE_PARAM, null, Value.class);
- Class<E> edgeClass = parameters.getClass(EDGE_PARAM, null, cl);
-
- vertexKey = InstantiationUtil.instantiate(vertexKeyClass, Key.class);
- vertexValue = InstantiationUtil.instantiate(vertexValueClass, Value.class);
-
- K edgeKeyHolder = InstantiationUtil.instantiate(vertexKeyClass, Key.class);
- E edgeValueHolder = InstantiationUtil.instantiate(edgeClass, Value.class);
-
- ClassLoader ucl = getRuntimeContext().getUserCodeClassLoader();
-
- try {
- this.messagingFunction = (MessagingFunction<K, V, M, E>) InstantiationUtil.readObjectFromConfig(parameters, UDF_PARAM, ucl);
- } catch (Exception e) {
- String message = e.getMessage() == null ? "." : ": " + e.getMessage();
- throw new Exception("Could not instantiate MessagingFunction" + message, e);
- }
-
- this.messagingFunction.init(getIterationRuntimeContext(), edgeKeyHolder, edgeValueHolder);
- this.messagingFunction.setup(parameters);
- }
- this.messagingFunction.preSuperstep();
- }
-
- @Override
- public void close() throws Exception {
- this.messagingFunction.postSuperstep();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/VertexUpdateFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/VertexUpdateFunction.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/VertexUpdateFunction.java
deleted file mode 100644
index 299b898..0000000
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/VertexUpdateFunction.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.spargel.java.record;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.common.aggregators.Aggregator;
-import org.apache.flink.api.common.functions.IterationRuntimeContext;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.types.Key;
-import org.apache.flink.types.Record;
-import org.apache.flink.types.Value;
-import org.apache.flink.util.Collector;
-
-/**
- *
- * <VertexKey> The vertex key type.
- * <VertexValue> The vertex value type.
- * <Message> The message type.
- */
-public abstract class VertexUpdateFunction<VertexKey extends Key<VertexKey>, VertexValue extends Value, Message extends Value> implements Serializable {
-
- // --------------------------------------------------------------------------------------------
- // Public API Methods
- // --------------------------------------------------------------------------------------------
-
- public abstract void updateVertex(VertexKey vertexKey, VertexValue vertexValue, MessageIterator<Message> inMessages) throws Exception;
-
- public void setup(Configuration config) throws Exception {}
-
- public void preSuperstep() throws Exception {}
-
- public void postSuperstep() throws Exception {}
-
- public void setNewVertexValue(VertexValue newValue) {
- outVal.setField(1, newValue);
- out.collect(outVal);
- }
-
- public int getSuperstep() {
- return this.runtimeContext.getSuperstepNumber();
- }
-
- public <T extends Aggregator<?>> T getIterationAggregator(String name) {
- return this.runtimeContext.<T>getIterationAggregator(name);
- }
-
- public <T extends Value> T getPreviousIterationAggregate(String name) {
- return this.runtimeContext.<T>getPreviousIterationAggregate(name);
- }
-
- // --------------------------------------------------------------------------------------------
- // internal methods
- // --------------------------------------------------------------------------------------------
-
- private IterationRuntimeContext runtimeContext;
-
- private Collector<Record> out;
-
- private Record outVal;
-
-
- void init(IterationRuntimeContext context) {
- this.runtimeContext = context;
- }
-
- void setOutput(Record val, Collector<Record> out) {
- this.out = out;
- this.outVal = val;
- }
-
- // serializability
- private static final long serialVersionUID = 1L;
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java b/flink-addons/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java
deleted file mode 100644
index 4b6afd3..0000000
--- a/flink-addons/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.spargel.java;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.junit.Test;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.compiler.plan.DualInputPlanNode;
-import org.apache.flink.compiler.plan.OptimizedPlan;
-import org.apache.flink.compiler.plan.PlanNode;
-import org.apache.flink.compiler.plan.SinkPlanNode;
-import org.apache.flink.compiler.plan.WorksetIterationPlanNode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.spargel.java.VertexCentricIteration;
-import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.CCMessager;
-import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.CCUpdater;
-import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.IdAssigner;
-import org.apache.flink.test.compiler.util.CompilerTestBase;
-
-
-public class SpargelCompilerTest extends CompilerTestBase {
-
- @Test
- public void testSpargelCompiler() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
- // compose test program
- {
- DataSet<Long> vertexIds = env.generateSequence(1, 2);
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
-
- DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
- DataSet<Tuple2<Long, Long>> result = initialVertices.runOperation(VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100));
-
- result.print();
- }
-
- Plan p = env.createProgramPlan("Spargel Connected Components");
- OptimizedPlan op = compileNoStats(p);
-
- // check the sink
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(DEFAULT_PARALLELISM, sink.getDegreeOfParallelism());
-
- // check the iteration
- WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
- assertEquals(DEFAULT_PARALLELISM, iteration.getDegreeOfParallelism());
-
- // check the solution set join and the delta
- PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
- assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
-
- DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
- assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
- assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
- assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
-
- // check the workset set join
- DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
- assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
- assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
- assertTrue(edgeJoin.getInput1().getTempMode().isCached());
-
- assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
-
- // check that the initial partitioning is pushed out of the loop
- assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
- assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
- assertEquals(new FieldList(0), iteration.getInput2().getShipStrategyKeys());
-
- // check that the initial workset sort is outside the loop
- assertEquals(LocalStrategy.SORT, iteration.getInput2().getLocalStrategy());
- assertEquals(new FieldList(0), iteration.getInput2().getLocalStrategyKeys());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testSpargelCompilerWithBroadcastVariable() {
- try {
- final String BC_VAR_NAME = "borat variable";
-
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
- // compose test program
- {
- DataSet<Long> bcVar = env.fromElements(1L);
-
- DataSet<Long> vertexIds = env.generateSequence(1, 2);
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
-
- DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
-
- VertexCentricIteration<Long, Long, Long, ?> vcIter = VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100);
- vcIter.addBroadcastSetForMessagingFunction(BC_VAR_NAME, bcVar);
- vcIter.addBroadcastSetForUpdateFunction(BC_VAR_NAME, bcVar);
-
- DataSet<Tuple2<Long, Long>> result = initialVertices.runOperation(vcIter);
-
- result.print();
- }
-
- Plan p = env.createProgramPlan("Spargel Connected Components");
- OptimizedPlan op = compileNoStats(p);
-
- // check the sink
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(DEFAULT_PARALLELISM, sink.getDegreeOfParallelism());
-
- // check the iteration
- WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
- assertEquals(DEFAULT_PARALLELISM, iteration.getDegreeOfParallelism());
-
- // check the solution set join and the delta
- PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
- assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
-
- DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
- assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
- assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
- assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
-
- // check the workset set join
- DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
- assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
- assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
- assertTrue(edgeJoin.getInput1().getTempMode().isCached());
-
- assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
-
- // check that the initial partitioning is pushed out of the loop
- assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
- assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
- assertEquals(new FieldList(0), iteration.getInput2().getShipStrategyKeys());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelTranslationTest.java b/flink-addons/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelTranslationTest.java
deleted file mode 100644
index b31618c..0000000
--- a/flink-addons/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelTranslationTest.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.spargel.java;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.junit.Test;
-import org.apache.flink.api.common.aggregators.LongSumAggregator;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.java.operators.DeltaIterationResultSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.TwoInputUdfOperator;
-import org.apache.flink.api.java.tuple.Tuple2;
-
-@SuppressWarnings("serial")
-public class SpargelTranslationTest {
-
- @Test
- public void testTranslationPlainEdges() {
- try {
- final String ITERATION_NAME = "Test Name";
-
- final String AGGREGATOR_NAME = "AggregatorName";
-
- final String BC_SET_MESSAGES_NAME = "borat messages";
-
- final String BC_SET_UPDATES_NAME = "borat updates";
- ;
- final int NUM_ITERATIONS = 13;
-
- final int ITERATION_DOP = 77;
-
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> bcMessaging = env.fromElements(1L);
- DataSet<Long> bcUpdate = env.fromElements(1L);
-
- DataSet<Tuple2<String, Double>> result;
-
- // ------------ construct the test program ------------------
- {
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<String, Double>> initialVertices = env.fromElements(new Tuple2<String, Double>("abc", 3.44));
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<String, String>> edges = env.fromElements(new Tuple2<String, String>("a", "c"));
-
-
- VertexCentricIteration<String, Double, Long, ?> vertexIteration =
- VertexCentricIteration.withPlainEdges(edges, new UpdateFunction(), new MessageFunctionNoEdgeValue(), NUM_ITERATIONS);
- vertexIteration.addBroadcastSetForMessagingFunction(BC_SET_MESSAGES_NAME, bcMessaging);
- vertexIteration.addBroadcastSetForUpdateFunction(BC_SET_UPDATES_NAME, bcUpdate);
-
- vertexIteration.setName(ITERATION_NAME);
- vertexIteration.setParallelism(ITERATION_DOP);
-
- vertexIteration.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());
-
- result = initialVertices.runOperation(vertexIteration);
- }
-
-
- // ------------- validate the java program ----------------
-
- assertTrue(result instanceof DeltaIterationResultSet);
-
- DeltaIterationResultSet<?, ?> resultSet = (DeltaIterationResultSet<?, ?>) result;
- DeltaIteration<?, ?> iteration = (DeltaIteration<?, ?>) resultSet.getIterationHead();
-
- // check the basic iteration properties
- assertEquals(NUM_ITERATIONS, resultSet.getMaxIterations());
- assertArrayEquals(new int[] {0}, resultSet.getKeyPositions());
- assertEquals(ITERATION_DOP, iteration.getParallelism());
- assertEquals(ITERATION_NAME, iteration.getName());
-
- assertEquals(AGGREGATOR_NAME, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());
-
- // validate that the semantic properties are set as they should
- TwoInputUdfOperator<?, ?, ?, ?> solutionSetJoin = (TwoInputUdfOperator<?, ?, ?, ?>) resultSet.getNextWorkset();
- assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(0, 0).contains(0));
- assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(1, 0).contains(0));
-
- TwoInputUdfOperator<?, ?, ?, ?> edgesJoin = (TwoInputUdfOperator<?, ?, ?, ?>) solutionSetJoin.getInput1();
-
- // validate that the broadcast sets are forwarded
- assertEquals(bcUpdate, solutionSetJoin.getBroadcastSets().get(BC_SET_UPDATES_NAME));
- assertEquals(bcMessaging, edgesJoin.getBroadcastSets().get(BC_SET_MESSAGES_NAME));
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testTranslationPlainEdgesWithForkedBroadcastVariable() {
- try {
- final String ITERATION_NAME = "Test Name";
-
- final String AGGREGATOR_NAME = "AggregatorName";
-
- final String BC_SET_MESSAGES_NAME = "borat messages";
-
- final String BC_SET_UPDATES_NAME = "borat updates";
- ;
- final int NUM_ITERATIONS = 13;
-
- final int ITERATION_DOP = 77;
-
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> bcVar = env.fromElements(1L);
-
- DataSet<Tuple2<String, Double>> result;
-
- // ------------ construct the test program ------------------
- {
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<String, Double>> initialVertices = env.fromElements(new Tuple2<String, Double>("abc", 3.44));
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<String, String>> edges = env.fromElements(new Tuple2<String, String>("a", "c"));
-
-
- VertexCentricIteration<String, Double, Long, ?> vertexIteration =
- VertexCentricIteration.withPlainEdges(edges, new UpdateFunction(), new MessageFunctionNoEdgeValue(), NUM_ITERATIONS);
- vertexIteration.addBroadcastSetForMessagingFunction(BC_SET_MESSAGES_NAME, bcVar);
- vertexIteration.addBroadcastSetForUpdateFunction(BC_SET_UPDATES_NAME, bcVar);
-
- vertexIteration.setName(ITERATION_NAME);
- vertexIteration.setParallelism(ITERATION_DOP);
-
- vertexIteration.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());
-
- result = initialVertices.runOperation(vertexIteration);
- }
-
-
- // ------------- validate the java program ----------------
-
- assertTrue(result instanceof DeltaIterationResultSet);
-
- DeltaIterationResultSet<?, ?> resultSet = (DeltaIterationResultSet<?, ?>) result;
- DeltaIteration<?, ?> iteration = (DeltaIteration<?, ?>) resultSet.getIterationHead();
-
- // check the basic iteration properties
- assertEquals(NUM_ITERATIONS, resultSet.getMaxIterations());
- assertArrayEquals(new int[] {0}, resultSet.getKeyPositions());
- assertEquals(ITERATION_DOP, iteration.getParallelism());
- assertEquals(ITERATION_NAME, iteration.getName());
-
- assertEquals(AGGREGATOR_NAME, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());
-
- // validate that the semantic properties are set as they should
- TwoInputUdfOperator<?, ?, ?, ?> solutionSetJoin = (TwoInputUdfOperator<?, ?, ?, ?>) resultSet.getNextWorkset();
- assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(0, 0).contains(0));
- assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(1, 0).contains(0));
-
- TwoInputUdfOperator<?, ?, ?, ?> edgesJoin = (TwoInputUdfOperator<?, ?, ?, ?>) solutionSetJoin.getInput1();
-
- // validate that the broadcast sets are forwarded
- assertEquals(bcVar, solutionSetJoin.getBroadcastSets().get(BC_SET_UPDATES_NAME));
- assertEquals(bcVar, edgesJoin.getBroadcastSets().get(BC_SET_MESSAGES_NAME));
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static class UpdateFunction extends VertexUpdateFunction<String, Double, Long> {
-
- @Override
- public void updateVertex(String vertexKey, Double vertexValue, MessageIterator<Long> inMessages) {}
- }
-
- public static class MessageFunctionNoEdgeValue extends MessagingFunction<String, Double, Long, Object> {
-
- @Override
- public void sendMessages(String vertexKey, Double vertexValue) {}
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java b/flink-addons/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java
deleted file mode 100644
index 166f7a2..0000000
--- a/flink-addons/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.test.spargel;
-
-import java.io.BufferedReader;
-
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.spargel.java.VertexCentricIteration;
-import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.CCMessager;
-import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.CCUpdater;
-import org.apache.flink.spargel.java.examples.SpargelConnectedComponents.IdAssigner;
-import org.apache.flink.test.testdata.ConnectedComponentsData;
-import org.apache.flink.test.util.JavaProgramTestBase;
-
-@SuppressWarnings("serial")
-public class SpargelConnectedComponentsITCase extends JavaProgramTestBase {
-
- private static final long SEED = 9487520347802987L;
-
- private static final int NUM_VERTICES = 1000;
-
- private static final int NUM_EDGES = 10000;
-
- private String resultPath;
-
-
- @Override
- protected void preSubmit() throws Exception {
- resultPath = getTempFilePath("results");
- }
-
- @Override
- protected void testProgram() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> vertexIds = env.generateSequence(1, NUM_VERTICES);
- DataSet<String> edgeString = env.fromElements(ConnectedComponentsData.getRandomOddEvenEdges(NUM_EDGES, NUM_VERTICES, SEED).split("\n"));
-
- DataSet<Tuple2<Long, Long>> edges = edgeString.map(new EdgeParser());
-
- DataSet<Tuple2<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());
- DataSet<Tuple2<Long, Long>> result = initialVertices.runOperation(VertexCentricIteration.withPlainEdges(edges, new CCUpdater(), new CCMessager(), 100));
-
- result.writeAsCsv(resultPath, "\n", " ");
- env.execute("Spargel Connected Components");
- }
-
- @Override
- protected void postSubmit() throws Exception {
- for (BufferedReader reader : getResultReader(resultPath)) {
- ConnectedComponentsData.checkOddEvenResult(reader);
- }
- }
-
- public static final class EdgeParser extends RichMapFunction<String, Tuple2<Long, Long>> {
- public Tuple2<Long, Long> map(String value) {
- String[] nums = value.split(" ");
- return new Tuple2<Long, Long>(Long.parseLong(nums[0]), Long.parseLong(nums[1]));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/test/resources/log4j-test.properties b/flink-addons/flink-spargel/src/test/resources/log4j-test.properties
deleted file mode 100644
index 2fb9345..0000000
--- a/flink-addons/flink-spargel/src/test/resources/log4j-test.properties
+++ /dev/null
@@ -1,19 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-spargel/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/test/resources/logback-test.xml b/flink-addons/flink-spargel/src/test/resources/logback-test.xml
deleted file mode 100644
index 8b3bb27..0000000
--- a/flink-addons/flink-spargel/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
- </encoder>
- </appender>
-
- <root level="WARN">
- <appender-ref ref="STDOUT"/>
- </root>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/pom.xml b/flink-addons/flink-streaming/flink-streaming-connectors/pom.xml
deleted file mode 100644
index df34b20..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/pom.xml
+++ /dev/null
@@ -1,243 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-parent</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-streaming-connectors</artifactId>
- <name>flink-streaming-connectors</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>0.8.0</version>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jmx</groupId>
- <artifactId>jmxri</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jdmk</groupId>
- <artifactId>jmxtools</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.sf.jopt-simple</groupId>
- <artifactId>jopt-simple</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-reflect</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-annotation</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>com.rabbitmq</groupId>
- <artifactId>amqp-client</artifactId>
- <version>3.3.1</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-core</artifactId>
- <version>1.5.0</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.tukaani</groupId>
- <artifactId>xz</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.velocity</groupId>
- <artifactId>velocity</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>hbc-core</artifactId>
- <version>2.2.0</version>
- </dependency>
-
- <dependency>
- <groupId>org.fusesource.leveldbjni</groupId>
- <artifactId>leveldbjni-all</artifactId>
- <version>1.8</version>
- </dependency>
-
- <dependency>
- <groupId>redis.clients</groupId>
- <artifactId>jedis</artifactId>
- <version>2.4.2</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.simple-spring-memcached</groupId>
- <artifactId>spymemcached</artifactId>
- <version>2.8.4</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.sling</groupId>
- <artifactId>org.apache.sling.commons.json</artifactId>
- <version>2.0.6</version>
- </dependency>
-
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
deleted file mode 100644
index 1623943..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.function.source.GenericSourceFunction;
-import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.connectors.util.DeserializationSchema;
-
-public abstract class ConnectorSource<OUT> extends RichParallelSourceFunction<OUT> implements
- GenericSourceFunction<OUT> {
-
- private static final long serialVersionUID = 1L;
- protected DeserializationSchema<OUT> schema;
-
- public ConnectorSource(DeserializationSchema<OUT> schema) {
- this.schema = schema;
- }
-
- @Override
- public TypeInformation<OUT> getType() {
- return TypeExtractor.createTypeInfo(DeserializationSchema.class, schema.getClass(), 0,
- null, null);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/CustomSerializationDBState.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/CustomSerializationDBState.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/CustomSerializationDBState.java
deleted file mode 100644
index 9b1de36..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/CustomSerializationDBState.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.db;
-
-import java.io.Serializable;
-
-public abstract class CustomSerializationDBState<K extends Serializable, V extends Serializable> {
-
- protected DBSerializer<K> keySerializer;
- protected DBSerializer<V> valueSerializer;
-
- public CustomSerializationDBState(DBSerializer<K> keySerializer, DBSerializer<V> valueSerializer) {
- this.keySerializer = keySerializer;
- this.valueSerializer = valueSerializer;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBSerializer.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBSerializer.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBSerializer.java
deleted file mode 100644
index a80e20c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBSerializer.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.flink.streaming.connectors.db;
-
-import java.io.Serializable;
-
-/**
- * Interface for custom serialization of keys and values used in external
- * databases.
- *
- * @param <T>
- * Type of the key or value to serialize
- */
-public interface DBSerializer<T extends Serializable> {
-
- byte[] write(T object);
-
- T read(byte[] serializedObject);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBState.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBState.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBState.java
deleted file mode 100644
index 7528182..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBState.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.db;
-
-public interface DBState<K, V> {
-
- public void put(K key, V value);
-
- public V get(K key);
-
- public void remove(K key);
-
- public void close();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateIterator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateIterator.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateIterator.java
deleted file mode 100644
index 890d933..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateIterator.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.db;
-
-import java.io.Serializable;
-
-public abstract class DBStateIterator<K extends Serializable, V extends Serializable> {
-
- public abstract boolean hasNext();
-
- public abstract K getNextKey();
-
- public abstract V getNextValue();
-
- public abstract void next();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateWithIterator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateWithIterator.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateWithIterator.java
deleted file mode 100644
index 3383330..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DBStateWithIterator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.db;
-
-import java.io.Serializable;
-
-public interface DBStateWithIterator<K extends Serializable, V extends Serializable> extends DBState<K, V> {
-
- public DBStateIterator<K, V> getIterator();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DefaultDBSerializer.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DefaultDBSerializer.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DefaultDBSerializer.java
deleted file mode 100644
index 6b770cb..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/DefaultDBSerializer.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.db;
-
-import java.io.Serializable;
-
-import org.apache.commons.lang3.SerializationUtils;
-
-public class DefaultDBSerializer<T extends Serializable> implements DBSerializer<T> {
-
- @Override
- public byte[] write(T object) {
- return SerializationUtils.serialize(object);
- }
-
- @Override
- public T read(byte[] serializedObject) {
- return SerializationUtils.deserialize(serializedObject);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/LevelDBState.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/LevelDBState.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/LevelDBState.java
deleted file mode 100644
index 2e8626d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/LevelDBState.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.db;
-
-import static org.fusesource.leveldbjni.JniDBFactory.factory;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.iq80.leveldb.DB;
-import org.iq80.leveldb.DBIterator;
-import org.iq80.leveldb.Options;
-
-/**
- * Interface to a LevelDB key-value store.
- *
- * @see <a href="https://code.google.com/p/leveldb/">https://code.google.com/p/leveldb/</a>
- *
- * @param <K>
- * Type of key
- * @param <V>
- * Type of value
- */
-public class LevelDBState<K extends Serializable, V extends Serializable> extends
- CustomSerializationDBState<K, V> implements DBStateWithIterator<K, V> {
-
- private DB database;
-
- public LevelDBState(String dbName, DBSerializer<K> keySerializer,
- DBSerializer<V> valueSerializer) {
- super(keySerializer, valueSerializer);
- Options options = new Options();
- File file = new File(dbName);
- options.createIfMissing(true);
- try {
- factory.destroy(file, options);
- database = factory.open(file, options);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public LevelDBState(String dbName) {
- this(dbName, new DefaultDBSerializer<K>(), new DefaultDBSerializer<V>());
- }
-
- @Override
- public void close() {
- try {
- database.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void put(K key, V value) {
- database.put(keySerializer.write(key), valueSerializer.write(value));
- }
-
- @Override
- public V get(K key) {
- byte[] serializedValue = database.get(keySerializer.write(key));
- if (serializedValue != null) {
- return valueSerializer.read(serializedValue);
- } else {
- throw new RuntimeException("No such entry at key " + key);
- }
- }
-
- @Override
- public void remove(K key) {
- database.delete(keySerializer.write(key));
- }
-
- @Override
- public DBStateIterator<K, V> getIterator() {
- return new LevelDBStateIterator();
- }
-
- private class LevelDBStateIterator extends DBStateIterator<K, V> {
- private DBIterator iterator;
-
- public LevelDBStateIterator() {
- this.iterator = database.iterator();
- this.iterator.seekToFirst();
- }
-
- @Override
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- @Override
- public K getNextKey() {
- return keySerializer.read(iterator.peekNext().getKey());
- }
-
- @Override
- public V getNextValue() {
- return valueSerializer.read(iterator.peekNext().getValue());
- }
-
- @Override
- public void next() {
- iterator.next();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/MemcachedState.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/MemcachedState.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/MemcachedState.java
deleted file mode 100644
index 7f06a1f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/MemcachedState.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.db;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import net.spy.memcached.MemcachedClient;
-
-/**
- * Interface to a Memcached key-value cache. It needs a running instance of Memcached.
- *
- * @see <a href="http://memcached.org/">http://memcached.org</a>
- *
- * @param <V>
- * Type of value
- */
-public class MemcachedState<V> implements DBState<String, V> {
-
- private MemcachedClient memcached;
-
- public MemcachedState() {
- try {
- memcached = new MemcachedClient(new InetSocketAddress("localhost", 11211));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public MemcachedState(String hostname, int portNum) {
- try {
- memcached = new MemcachedClient(new InetSocketAddress(hostname, portNum));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void close() {
- memcached.shutdown();
- }
-
- @Override
- public void put(String key, V value) {
- memcached.set(key, 0, value);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public V get(String key) {
- return (V) memcached.get(key);
- }
-
- @Override
- public void remove(String key) {
- memcached.delete(key.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/RedisState.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/RedisState.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/RedisState.java
deleted file mode 100644
index b9244c5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/db/RedisState.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.db;
-
-import java.io.Serializable;
-import java.util.Iterator;
-import java.util.Set;
-
-import redis.clients.jedis.Jedis;
-
-/**
- * Interface to a Redis key-value cache. It needs a running instance of Redis.
- *
- * @see <a href="http://redis.io/">http://redis.io/</a>
- *
- * @param <K>
- * Type of key
- * @param <V>
- * Type of value
- */
-public class RedisState<K extends Serializable, V extends Serializable> extends
- CustomSerializationDBState<K, V> implements DBStateWithIterator<K, V> {
-
- private Jedis jedis;
-
- public RedisState(DBSerializer<K> keySerializer, DBSerializer<V> valueSerializer) {
- super(keySerializer, valueSerializer);
- jedis = new Jedis("localhost");
- }
-
- public RedisState() {
- this(new DefaultDBSerializer<K>(), new DefaultDBSerializer<V>());
- }
-
- @Override
- public void close() {
- jedis.close();
- }
-
- @Override
- public void put(K key, V value) {
- jedis.set(keySerializer.write(key), valueSerializer.write(value));
- }
-
- @Override
- public V get(K key) {
- return valueSerializer.read(jedis.get(keySerializer.write(key)));
- }
-
- @Override
- public void remove(K key) {
- jedis.del(keySerializer.write(key));
- }
-
- @Override
- public DBStateIterator<K, V> getIterator() {
- return new RedisStateIterator();
- }
-
- private class RedisStateIterator extends DBStateIterator<K, V> {
-
- private Set<byte[]> set;
- private Iterator<byte[]> iterator;
- private byte[] currentKey;
-
- public RedisStateIterator() {
- set = jedis.keys(new byte[0]);
- jedis.keys("*".getBytes()).iterator();
- iterator = set.iterator();
- currentKey = iterator.next();
- }
-
- @Override
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- @Override
- public K getNextKey() {
- return keySerializer.read(currentKey);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public V getNextValue() {
- return (V) jedis.get(currentKey);
- }
-
- @Override
- public void next() {
- currentKey = iterator.next();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
deleted file mode 100644
index 8a2f2b8..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.flume;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
-import org.apache.flink.streaming.connectors.util.SerializationSchema;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.apache.flume.api.RpcClient;
-import org.apache.flume.api.RpcClientFactory;
-import org.apache.flume.event.EventBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FlumeSink<IN> extends RichSinkFunction<IN> {
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(FlumeSink.class);
-
- private transient FlinkRpcClientFacade client;
- boolean initDone = false;
- String host;
- int port;
- SerializationSchema<IN, byte[]> scheme;
-
- public FlumeSink(String host, int port, SerializationSchema<IN, byte[]> schema) {
- this.host = host;
- this.port = port;
- this.scheme = schema;
- }
-
- /**
- * Receives tuples from the Apache Flink {@link DataStream} and forwards
- * them to Apache Flume.
- *
- * @param value
- * The tuple arriving from the datastream
- */
- @Override
- public void invoke(IN value) {
-
- byte[] data = scheme.serialize(value);
- client.sendDataToFlume(data);
-
- }
-
- private class FlinkRpcClientFacade {
- private RpcClient client;
- private String hostname;
- private int port;
-
- /**
- * Initializes the connection to Apache Flume.
- *
- * @param hostname
- * The host
- * @param port
- * The port.
- */
- public void init(String hostname, int port) {
- // Setup the RPC connection
- this.hostname = hostname;
- this.port = port;
- int initCounter = 0;
- while (true) {
- if (initCounter >= 90) {
- throw new RuntimeException("Cannot establish connection with" + port + " at "
- + host);
- }
- try {
- this.client = RpcClientFactory.getDefaultInstance(hostname, port);
- } catch (FlumeException e) {
- // Wait one second if the connection failed before the next
- // try
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e1) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Interrupted while trying to connect {} at {}", port, host);
- }
- }
- }
- if (client != null) {
- break;
- }
- initCounter++;
- }
- initDone = true;
- }
-
- /**
- * Sends byte arrays as {@link Event} series to Apache Flume.
- *
- * @param data
- * The byte array to send to Apache FLume
- */
- public void sendDataToFlume(byte[] data) {
- Event event = EventBuilder.withBody(data);
-
- try {
- client.append(event);
-
- } catch (EventDeliveryException e) {
- // clean up and recreate the client
- client.close();
- client = null;
- client = RpcClientFactory.getDefaultInstance(hostname, port);
- }
- }
-
- }
-
- @Override
- public void close() {
- client.client.close();
- }
-
- @Override
- public void open(Configuration config) {
- client = new FlinkRpcClientFacade();
- client.init(host, port);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
deleted file mode 100644
index 4f6ec2d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.flume;
-
-import java.util.List;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.connectors.ConnectorSource;
-import org.apache.flink.streaming.connectors.util.DeserializationSchema;
-import org.apache.flink.util.Collector;
-import org.apache.flume.Context;
-import org.apache.flume.channel.ChannelProcessor;
-import org.apache.flume.source.AvroSource;
-import org.apache.flume.source.avro.AvroFlumeEvent;
-import org.apache.flume.source.avro.Status;
-
-public class FlumeSource<OUT> extends ConnectorSource<OUT> {
- private static final long serialVersionUID = 1L;
-
- String host;
- String port;
- volatile boolean finished = false;
-
- FlumeSource(String host, int port, DeserializationSchema<OUT> deserializationSchema) {
- super(deserializationSchema);
- this.host = host;
- this.port = Integer.toString(port);
- }
-
- public class MyAvroSource extends AvroSource {
- Collector<OUT> collector;
-
- /**
- * Sends the AvroFlumeEvent from it's argument list to the Apache Flink
- * {@link DataStream}.
- *
- * @param avroEvent
- * The event that should be sent to the dataStream
- * @return A {@link Status}.OK message if sending the event was
- * successful.
- */
- @Override
- public Status append(AvroFlumeEvent avroEvent) {
- collect(avroEvent);
- return Status.OK;
- }
-
- /**
- * Sends the AvroFlumeEvents from it's argument list to the Apache Flink
- * {@link DataStream}.
- *
- * @param events
- * The events that is sent to the dataStream
- * @return A Status.OK message if sending the events was successful.
- */
- @Override
- public Status appendBatch(List<AvroFlumeEvent> events) {
- for (AvroFlumeEvent avroEvent : events) {
- collect(avroEvent);
- }
-
- return Status.OK;
- }
-
- /**
- * Deserializes the AvroFlumeEvent before sending it to the Apache Flink
- * {@link DataStream}.
- *
- * @param avroEvent
- * The event that is sent to the dataStream
- */
- private void collect(AvroFlumeEvent avroEvent) {
- byte[] b = avroEvent.getBody().array();
- OUT out = FlumeSource.this.schema.deserialize(b);
-
- if (schema.isEndOfStream(out)) {
- FlumeSource.this.finished = true;
- this.stop();
- FlumeSource.this.notifyAll();
- } else {
- collector.collect(out);
- }
-
- }
-
- }
-
- MyAvroSource avroSource;
-
- /**
- * Configures the AvroSource. Also sets the collector so the application can
- * use it from outside of the invoke function.
- *
- * @param collector
- * The collector used in the invoke function
- */
- public void configureAvroSource(Collector<OUT> collector) {
-
- avroSource = new MyAvroSource();
- avroSource.collector = collector;
- Context context = new Context();
- context.put("port", port);
- context.put("bind", host);
- avroSource.configure(context);
- // An instance of a ChannelProcessor is required for configuring the
- // avroSource although it will not be used in this case.
- ChannelProcessor cp = new ChannelProcessor(null);
- avroSource.setChannelProcessor(cp);
- }
-
- /**
- * Configures the AvroSource and runs until the user calls a close function.
- *
- * @param collector
- * The Collector for sending data to the datastream
- */
- @Override
- public void invoke(Collector<OUT> collector) throws Exception {
- configureAvroSource(collector);
- avroSource.start();
- while (!finished) {
- this.wait();
- }
- }
-
-}
[33/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/MapCheckpoint.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/MapCheckpoint.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/MapCheckpoint.java
deleted file mode 100644
index 15d1fd5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/MapCheckpoint.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state.checkpoint;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.streaming.state.MapState;
-import org.apache.flink.streaming.state.OperatorState;
-
-public class MapCheckpoint<K, V> extends StateCheckpoint<Map<K, V>> {
-
- private static final long serialVersionUID = 1L;
-
- protected Set<K> removedItems;
- protected Map<K, V> updatedItems;
- protected boolean clear;
-
- @SuppressWarnings("unchecked")
- public MapCheckpoint(OperatorState<Map<K, V>> operatorState) {
- if (operatorState instanceof MapState) {
- MapState<K, V> mapState = (MapState<K, V>) operatorState;
-
- this.removedItems = mapState.getRemovedItems();
- this.clear = mapState.isCleared();
-
- this.updatedItems = new HashMap<K, V>();
- for (K key : mapState.getUpdatedItems()) {
- this.updatedItems.put(key, mapState.get(key));
- }
- this.checkpointedState = this.updatedItems;
-
- } else {
- throw new RuntimeException("MapCheckpoint can only be used with MapState");
- }
-
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public StateCheckpoint<Map<K, V>> update(StateCheckpoint<Map<K, V>> nextCheckpoint) {
- MapCheckpoint<K, V> mapCheckpoint = (MapCheckpoint<K, V>) nextCheckpoint;
- if (this.checkpointedState == null) {
- this.checkpointedState = mapCheckpoint.updatedItems;
- } else {
- if (mapCheckpoint.clear) {
- this.checkpointedState.clear();
- }
- for (Object key : mapCheckpoint.removedItems) {
- this.checkpointedState.remove(key);
- }
- this.checkpointedState.putAll(mapCheckpoint.updatedItems);
- }
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/StateCheckpoint.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/StateCheckpoint.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/StateCheckpoint.java
deleted file mode 100644
index 8b76245..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/state/checkpoint/StateCheckpoint.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.state.checkpoint;
-
-import java.io.Serializable;
-
-import org.apache.flink.streaming.state.OperatorState;
-
-/**
- * Base class for creating checkpoints for {@link OperatorState}. This
- * checkpoints will be used to backup states in stateful Flink operators and
- * also to restore them in case of node failure. To allow incremental
- * checkpoints override the {@link #update(StateCheckpoint)} method.
- *
- * @param <T>
- * The type of the state.
- */
-public class StateCheckpoint<T> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- T checkpointedState;
-
- /**
- * Creates a state checkpoint from the given {@link OperatorState}
- *
- * @param operatorState
- * The {@link OperatorState} to checkpoint.
- */
- public StateCheckpoint(OperatorState<T> operatorState) {
- this.checkpointedState = operatorState.getState();
- }
-
- public StateCheckpoint() {
- this.checkpointedState = null;
- }
-
- /**
- * Returns the state object for the checkpoint.
- *
- * @return The checkpointed state object.
- */
- public T getCheckpointedState() {
- return checkpointedState;
- }
-
- /**
- * Updates the checkpoint from next one. Override this method to allow
- * incremental updates.
- *
- * @param nextCheckpoint
- * The {@link StateCheckpoint} will be used to update from.
- */
- public StateCheckpoint<T> update(StateCheckpoint<T> nextCheckpoint) {
- this.checkpointedState = nextCheckpoint.getCheckpointedState();
- return this;
- }
-
- @Override
- public String toString() {
- return checkpointedState.toString();
- }
-
- public boolean stateEquals(StateCheckpoint<T> other) {
- return checkpointedState.equals(other.checkpointedState);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/ClusterUtil.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/ClusterUtil.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/ClusterUtil.java
deleted file mode 100755
index 691b111..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/ClusterUtil.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.client.JobClient;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import akka.actor.ActorRef;
-
-public class ClusterUtil {
-
- private static final Logger LOG = LoggerFactory.getLogger(ClusterUtil.class);
- public static final String CANNOT_EXECUTE_EMPTY_JOB = "Cannot execute empty job";
-
- /**
- * Executes the given JobGraph locally, on a FlinkMiniCluster
- *
- * @param jobGraph
- * jobGraph
- * @param degreeOfParallelism
- * numberOfTaskTrackers
- * @param memorySize
- * memorySize
- */
- public static void runOnMiniCluster(JobGraph jobGraph, int degreeOfParallelism, long memorySize)
- throws Exception {
-
- Configuration configuration = jobGraph.getJobConfiguration();
-
- LocalFlinkMiniCluster exec = null;
-
- configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, memorySize);
- configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, degreeOfParallelism);
- if (LOG.isInfoEnabled()) {
- LOG.info("Running on mini cluster");
- }
-
- try {
- exec = new LocalFlinkMiniCluster(configuration, true);
- ActorRef jobClient = exec.getJobClient();
-
- JobClient.submitJobAndWait(jobGraph, true, jobClient, exec.timeout());
-
- } catch (Exception e) {
- throw e;
- } finally {
- if (exec != null) {
- exec.stop();
- }
- }
- }
-
- public static void runOnMiniCluster(JobGraph jobGraph, int numOfSlots) throws Exception {
- runOnMiniCluster(jobGraph, numOfSlots, -1);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
deleted file mode 100644
index 73e6360..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util.keys;
-
-import java.lang.reflect.Array;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.CompositeType;
-import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.operators.Keys;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple10;
-import org.apache.flink.api.java.tuple.Tuple11;
-import org.apache.flink.api.java.tuple.Tuple12;
-import org.apache.flink.api.java.tuple.Tuple13;
-import org.apache.flink.api.java.tuple.Tuple14;
-import org.apache.flink.api.java.tuple.Tuple15;
-import org.apache.flink.api.java.tuple.Tuple16;
-import org.apache.flink.api.java.tuple.Tuple17;
-import org.apache.flink.api.java.tuple.Tuple18;
-import org.apache.flink.api.java.tuple.Tuple19;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple20;
-import org.apache.flink.api.java.tuple.Tuple21;
-import org.apache.flink.api.java.tuple.Tuple22;
-import org.apache.flink.api.java.tuple.Tuple23;
-import org.apache.flink.api.java.tuple.Tuple24;
-import org.apache.flink.api.java.tuple.Tuple25;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.api.java.tuple.Tuple6;
-import org.apache.flink.api.java.tuple.Tuple7;
-import org.apache.flink.api.java.tuple.Tuple8;
-import org.apache.flink.api.java.tuple.Tuple9;
-
-public class KeySelectorUtil {
-
- public static Class<?>[] tupleClasses = new Class[] { Tuple1.class, Tuple2.class, Tuple3.class,
- Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class, Tuple9.class,
- Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class, Tuple14.class,
- Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class, Tuple19.class,
- Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class, Tuple24.class,
- Tuple25.class };
-
- public static <X> KeySelector<X, ?> getSelectorForKeys(Keys<X> keys, TypeInformation<X> typeInfo) {
- int[] logicalKeyPositions = keys.computeLogicalKeyPositions();
- int keyLength = logicalKeyPositions.length;
- boolean[] orders = new boolean[keyLength];
- TypeComparator<X> comparator = ((CompositeType<X>) typeInfo).createComparator(
- logicalKeyPositions, orders, 0);
- return new ComparableKeySelector<X>(comparator, keyLength);
- }
-
- public static class ComparableKeySelector<IN> implements KeySelector<IN, Tuple> {
-
- private static final long serialVersionUID = 1L;
-
- private TypeComparator<IN> comparator;
- private int keyLength;
- private Object[] keyArray;
- private Tuple key;
-
- public ComparableKeySelector(TypeComparator<IN> comparator, int keyLength) {
- this.comparator = comparator;
- this.keyLength = keyLength;
- keyArray = new Object[keyLength];
- try {
- key = (Tuple) tupleClasses[keyLength - 1].newInstance();
- } catch (Exception e) {
- }
- }
-
- @Override
- public Tuple getKey(IN value) throws Exception {
- comparator.extractKeys(value, keyArray, 0);
- for (int i = 0; i < keyLength; i++) {
- key.setField(keyArray[i], i);
- }
- return key;
- }
-
- }
-
- public static class ArrayKeySelector<IN> implements KeySelector<IN, Tuple> {
-
- private static final long serialVersionUID = 1L;
-
- Tuple key;
- int[] fields;
-
- public ArrayKeySelector(int... fields) {
- this.fields = fields;
- try {
- key = (Tuple) tupleClasses[fields.length - 1].newInstance();
- } catch (Exception e) {
- }
- }
-
- @Override
- public Tuple getKey(IN value) throws Exception {
- for (int i = 0; i < fields.length; i++) {
- int pos = fields[i];
- key.setField(Array.get(value, fields[pos]), i);
- }
- return key;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
deleted file mode 100644
index 49cd497..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
+++ /dev/null
@@ -1,318 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.operators.Keys;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
-import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
-import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
-import org.apache.flink.streaming.api.invokable.operator.GroupedReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.StreamReduceInvokable;
-import org.apache.flink.streaming.util.MockContext;
-import org.apache.flink.streaming.util.keys.KeySelectorUtil;
-import org.junit.Test;
-
-public class AggregationFunctionTest {
-
- @Test
- public void groupSumIntegerTest() {
-
- List<Tuple2<Integer, Integer>> expectedSumList = new ArrayList<Tuple2<Integer, Integer>>();
- List<Tuple2<Integer, Integer>> expectedMinList = new ArrayList<Tuple2<Integer, Integer>>();
- List<Tuple2<Integer, Integer>> expectedMaxList = new ArrayList<Tuple2<Integer, Integer>>();
- List<Integer> expectedSumList0 = new ArrayList<Integer>();
- List<Integer> expectedMinList0 = new ArrayList<Integer>();
- List<Integer> expectedMaxList0 = new ArrayList<Integer>();
- List<Tuple2<Integer, Integer>> expectedGroupSumList = new ArrayList<Tuple2<Integer, Integer>>();
- List<Tuple2<Integer, Integer>> expectedGroupMinList = new ArrayList<Tuple2<Integer, Integer>>();
- List<Tuple2<Integer, Integer>> expectedGroupMaxList = new ArrayList<Tuple2<Integer, Integer>>();
-
- List<Integer> simpleInput = new ArrayList<Integer>();
-
- int groupedSum0 = 0;
- int groupedSum1 = 0;
- int groupedSum2 = 0;
-
- for (int i = 0; i < 9; i++) {
- simpleInput.add(i);
- expectedSumList.add(new Tuple2<Integer, Integer>(i % 3, (i + 1) * i / 2));
- expectedMinList.add(new Tuple2<Integer, Integer>(i % 3, 0));
- expectedMaxList.add(new Tuple2<Integer, Integer>(i % 3, i));
-
- expectedSumList0.add((i + 1) * i / 2);
- expectedMaxList0.add(i);
- expectedMinList0.add(0);
-
- int groupedSum;
- switch (i % 3) {
- case 0:
- groupedSum = groupedSum0 += i;
- break;
- case 1:
- groupedSum = groupedSum1 += i;
- break;
- default:
- groupedSum = groupedSum2 += i;
- break;
- }
-
- expectedGroupSumList.add(new Tuple2<Integer, Integer>(i % 3, groupedSum));
- expectedGroupMinList.add(new Tuple2<Integer, Integer>(i % 3, i % 3));
- expectedGroupMaxList.add(new Tuple2<Integer, Integer>(i % 3, i));
- }
-
- TypeInformation<Tuple2<Integer, Integer>> type1 = TypeExtractor
- .getForObject(new Tuple2<Integer, Integer>(0, 0));
- TypeInformation<Integer> type2 = TypeExtractor.getForObject(2);
-
- ReduceFunction<Tuple2<Integer, Integer>> sumFunction = SumAggregator.getSumFunction(1,
- Integer.class, type1);
- ReduceFunction<Integer> sumFunction0 = SumAggregator
- .getSumFunction(0, Integer.class, type2);
- ReduceFunction<Tuple2<Integer, Integer>> minFunction = ComparableAggregator.getAggregator(
- 1, type1, AggregationType.MIN);
- ReduceFunction<Integer> minFunction0 = ComparableAggregator.getAggregator(0, type2,
- AggregationType.MIN);
- ReduceFunction<Tuple2<Integer, Integer>> maxFunction = ComparableAggregator.getAggregator(
- 1, type1, AggregationType.MAX);
- ReduceFunction<Integer> maxFunction0 = ComparableAggregator.getAggregator(0, type2,
- AggregationType.MAX);
- List<Tuple2<Integer, Integer>> sumList = MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(sumFunction), getInputList());
-
- List<Tuple2<Integer, Integer>> minList = MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(minFunction), getInputList());
-
- List<Tuple2<Integer, Integer>> maxList = MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxFunction), getInputList());
-
- TypeInformation<Tuple2<Integer, Integer>> typeInfo = TypeExtractor
- .getForObject(new Tuple2<Integer, Integer>(1, 1));
-
- KeySelector<Tuple2<Integer, Integer>, ?> keySelector = KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys<Tuple2<Integer, Integer>>(new int[] { 0 }, typeInfo),
- typeInfo);
-
- List<Tuple2<Integer, Integer>> groupedSumList = MockContext.createAndExecute(
- new GroupedReduceInvokable<Tuple2<Integer, Integer>>(sumFunction, keySelector),
- getInputList());
-
- List<Tuple2<Integer, Integer>> groupedMinList = MockContext.createAndExecute(
- new GroupedReduceInvokable<Tuple2<Integer, Integer>>(minFunction, keySelector),
- getInputList());
-
- List<Tuple2<Integer, Integer>> groupedMaxList = MockContext.createAndExecute(
- new GroupedReduceInvokable<Tuple2<Integer, Integer>>(maxFunction, keySelector),
- getInputList());
-
- assertEquals(expectedSumList, sumList);
- assertEquals(expectedMinList, minList);
- assertEquals(expectedMaxList, maxList);
- assertEquals(expectedGroupSumList, groupedSumList);
- assertEquals(expectedGroupMinList, groupedMinList);
- assertEquals(expectedGroupMaxList, groupedMaxList);
- assertEquals(expectedSumList0, MockContext.createAndExecute(
- new StreamReduceInvokable<Integer>(sumFunction0), simpleInput));
- assertEquals(expectedMinList0, MockContext.createAndExecute(
- new StreamReduceInvokable<Integer>(minFunction0), simpleInput));
- assertEquals(expectedMaxList0, MockContext.createAndExecute(
- new StreamReduceInvokable<Integer>(maxFunction0), simpleInput));
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
- try {
- env.generateSequence(1, 100).min(1);
- fail();
- } catch (Exception e) {
- // Nothing to do here
- }
- try {
- env.generateSequence(1, 100).min(2);
- fail();
- } catch (Exception e) {
- // Nothing to do here
- }
- try {
- env.generateSequence(1, 100).min(3);
- fail();
- } catch (Exception e) {
- // Nothing to do here
- }
-
- ReduceFunction<Tuple2<Integer, Integer>> maxByFunctionFirst = ComparableAggregator
- .getAggregator(0, type1, AggregationType.MAXBY, true);
- ReduceFunction<Tuple2<Integer, Integer>> maxByFunctionLast = ComparableAggregator
- .getAggregator(0, type1, AggregationType.MAXBY, false);
-
- ReduceFunction<Tuple2<Integer, Integer>> minByFunctionFirst = ComparableAggregator
- .getAggregator(0, type1, AggregationType.MINBY, true);
- ReduceFunction<Tuple2<Integer, Integer>> minByFunctionLast = ComparableAggregator
- .getAggregator(0, type1, AggregationType.MINBY, false);
-
- List<Tuple2<Integer, Integer>> maxByFirstExpected = new ArrayList<Tuple2<Integer, Integer>>();
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(1, 1));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
-
- List<Tuple2<Integer, Integer>> maxByLastExpected = new ArrayList<Tuple2<Integer, Integer>>();
- maxByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(1, 1));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 8));
-
- List<Tuple2<Integer, Integer>> minByFirstExpected = new ArrayList<Tuple2<Integer, Integer>>();
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
-
- List<Tuple2<Integer, Integer>> minByLastExpected = new ArrayList<Tuple2<Integer, Integer>>();
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
-
- assertEquals(maxByFirstExpected, MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxByFunctionFirst),
- getInputList()));
- assertEquals(maxByLastExpected, MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxByFunctionLast),
- getInputList()));
- assertEquals(minByLastExpected, MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(minByFunctionLast),
- getInputList()));
- assertEquals(minByFirstExpected, MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(minByFunctionFirst),
- getInputList()));
-
- }
-
- @Test
- public void minMaxByTest() {
- TypeInformation<Tuple2<Integer, Integer>> type1 = TypeExtractor
- .getForObject(new Tuple2<Integer, Integer>(0, 0));
-
- ReduceFunction<Tuple2<Integer, Integer>> maxByFunctionFirst = ComparableAggregator
- .getAggregator(0, type1, AggregationType.MAXBY, true);
- ReduceFunction<Tuple2<Integer, Integer>> maxByFunctionLast = ComparableAggregator
- .getAggregator(0, type1, AggregationType.MAXBY, false);
-
- ReduceFunction<Tuple2<Integer, Integer>> minByFunctionFirst = ComparableAggregator
- .getAggregator(0, type1, AggregationType.MINBY, true);
- ReduceFunction<Tuple2<Integer, Integer>> minByFunctionLast = ComparableAggregator
- .getAggregator(0, type1, AggregationType.MINBY, false);
-
- List<Tuple2<Integer, Integer>> maxByFirstExpected = new ArrayList<Tuple2<Integer, Integer>>();
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(1, 1));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByFirstExpected.add(new Tuple2<Integer, Integer>(2, 2));
-
- List<Tuple2<Integer, Integer>> maxByLastExpected = new ArrayList<Tuple2<Integer, Integer>>();
- maxByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(1, 1));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 2));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 5));
- maxByLastExpected.add(new Tuple2<Integer, Integer>(2, 8));
-
- List<Tuple2<Integer, Integer>> minByFirstExpected = new ArrayList<Tuple2<Integer, Integer>>();
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByFirstExpected.add(new Tuple2<Integer, Integer>(0, 0));
-
- List<Tuple2<Integer, Integer>> minByLastExpected = new ArrayList<Tuple2<Integer, Integer>>();
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 0));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 3));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
- minByLastExpected.add(new Tuple2<Integer, Integer>(0, 6));
-
- assertEquals(maxByFirstExpected, MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxByFunctionFirst),
- getInputList()));
- assertEquals(maxByLastExpected, MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxByFunctionLast),
- getInputList()));
- assertEquals(minByLastExpected, MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(minByFunctionLast),
- getInputList()));
- assertEquals(minByFirstExpected, MockContext.createAndExecute(
- new StreamReduceInvokable<Tuple2<Integer, Integer>>(minByFunctionFirst),
- getInputList()));
- }
-
- private List<Tuple2<Integer, Integer>> getInputList() {
- ArrayList<Tuple2<Integer, Integer>> inputList = new ArrayList<Tuple2<Integer, Integer>>();
- for (int i = 0; i < 9; i++) {
- inputList.add(new Tuple2<Integer, Integer>(i % 3, i));
- }
- return inputList;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java
deleted file mode 100644
index 6ad827a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.IterativeDataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.util.TestStreamEnvironment;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class IterateTest {
-
- private static final long MEMORYSIZE = 32;
- private static boolean iterated = false;
-
- public static final class IterationHead extends RichFlatMapFunction<Boolean, Boolean> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(Boolean value, Collector<Boolean> out) throws Exception {
- if (value) {
- iterated = true;
- } else {
- out.collect(value);
- }
-
- }
-
- }
-
- public static final class IterationTail extends RichFlatMapFunction<Boolean, Boolean> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(Boolean value, Collector<Boolean> out) throws Exception {
- out.collect(true);
-
- }
-
- }
-
- public static final class MySink implements SinkFunction<Boolean> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(Boolean tuple) {
- }
-
- }
-
- @Test
- public void test() throws Exception {
- StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
-
- env.setBufferTimeout(10);
-
- DataStream<Boolean> source = env.fromElements(false, false, false);
-
- IterativeDataStream<Boolean> iteration = source.iterate(3000);
-
- DataStream<Boolean> increment = iteration.flatMap(new IterationHead()).flatMap(
- new IterationTail());
-
- iteration.closeWith(increment).addSink(new MySink());
-
- env.execute();
-
- assertTrue(iterated);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java
deleted file mode 100644
index 2486715..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/OutputSplitterTest.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.util.TestStreamEnvironment;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-public class OutputSplitterTest {
-
- private static final long MEMORYSIZE = 32;
-
- private static ArrayList<Integer> splitterResult1 = new ArrayList<Integer>();
- private static ArrayList<Integer> splitterResult2 = new ArrayList<Integer>();
-
-
- private static ArrayList<Integer> expectedSplitterResult = new ArrayList<Integer>();
-
- @SuppressWarnings("unchecked")
- @Test
- public void testOnMergedDataStream() throws Exception {
- splitterResult1.clear();
- splitterResult2.clear();
-
- StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
- env.setBufferTimeout(1);
-
- DataStream<Integer> d1 = env.fromElements(0,2,4,6,8);
- DataStream<Integer> d2 = env.fromElements(1,3,5,7,9);
-
- d1 = d1.merge(d2);
-
- d1.split(new OutputSelector<Integer>() {
- private static final long serialVersionUID = 8354166915727490130L;
-
- @Override
- public Iterable<String> select(Integer value) {
- List<String> s = new ArrayList<String>();
- if (value > 4) {
- s.add(">");
- } else {
- s.add("<");
- }
- return s;
- }
- }).select(">").addSink(new SinkFunction<Integer>() {
-
- private static final long serialVersionUID = 5827187510526388104L;
-
- @Override
- public void invoke(Integer value) {
- splitterResult1.add(value);
- }
- });
-
- d1.split(new OutputSelector<Integer>() {
- private static final long serialVersionUID = -6822487543355994807L;
-
- @Override
- public Iterable<String> select(Integer value) {
- List<String> s = new ArrayList<String>();
- if (value % 3 == 0) {
- s.add("yes");
- } else {
- s.add("no");
- }
- return s;
- }
- }).select("yes").addSink(new SinkFunction<Integer>() {
- private static final long serialVersionUID = -2674335071267854599L;
-
- @Override
- public void invoke(Integer value) {
- splitterResult2.add(value);
- }
- });
- env.execute();
-
- Collections.sort(splitterResult1);
- Collections.sort(splitterResult2);
-
- expectedSplitterResult.clear();
- expectedSplitterResult.addAll(Arrays.asList(5,6,7,8,9));
- assertEquals(expectedSplitterResult, splitterResult1);
-
- expectedSplitterResult.clear();
- expectedSplitterResult.addAll(Arrays.asList(0,3,6,9));
- assertEquals(expectedSplitterResult, splitterResult2);
- }
-
- @Test
- public void testOnSingleDataStream() throws Exception {
- splitterResult1.clear();
- splitterResult2.clear();
-
- StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
- env.setBufferTimeout(1);
-
- DataStream<Integer> ds = env.fromElements(0,1,2,3,4,5,6,7,8,9);
-
- ds.split(new OutputSelector<Integer>() {
- private static final long serialVersionUID = 2524335410904414121L;
-
- @Override
- public Iterable<String> select(Integer value) {
- List<String> s = new ArrayList<String>();
- if (value % 2 == 0) {
- s.add("even");
- } else {
- s.add("odd");
- }
- return s;
- }
- }).select("even").addSink(new SinkFunction<Integer>() {
-
- private static final long serialVersionUID = -2995092337537209535L;
-
- @Override
- public void invoke(Integer value) {
- splitterResult1.add(value);
- }
- });
-
- ds.split(new OutputSelector<Integer>() {
-
- private static final long serialVersionUID = -511693919586034092L;
-
- @Override
- public Iterable<String> select(Integer value) {
- List<String> s = new ArrayList<String>();
- if (value % 4 == 0) {
- s.add("yes");
- } else {
- s.add("no");
- }
- return s;
- }
- }).select("yes").addSink(new SinkFunction<Integer>() {
-
- private static final long serialVersionUID = -1749077049727705424L;
-
- @Override
- public void invoke(Integer value) {
- splitterResult2.add(value);
- }
- });
- env.execute();
-
- Collections.sort(splitterResult1);
- Collections.sort(splitterResult2);
-
- expectedSplitterResult.clear();
- expectedSplitterResult.addAll(Arrays.asList(0,2,4,6,8));
- assertEquals(expectedSplitterResult, splitterResult1);
-
- expectedSplitterResult.clear();
- expectedSplitterResult.addAll(Arrays.asList(0,4,8));
- assertEquals(expectedSplitterResult, splitterResult2);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
deleted file mode 100755
index 757f6f6..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.TestStreamEnvironment;
-import org.junit.Test;
-
-public class PrintTest implements Serializable {
-
- private static final long serialVersionUID = 1L;
- private static final long MEMORYSIZE = 32;
-
- private static final class IdentityMap implements MapFunction<Long, Long> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Long map(Long value) throws Exception {
- return value;
- }
- }
-
- private static final class FilterAll implements FilterFunction<Long> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public boolean filter(Long value) throws Exception {
- return true;
- }
- }
-
- @Test
- public void test() throws Exception {
- StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
- env.generateSequence(1, 10).map(new IdentityMap()).filter(new FilterAll()).print();
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java
deleted file mode 100644
index 9be7de6..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.ByteArrayInputStream;
-import java.net.Socket;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.api.function.source.FromElementsFunction;
-import org.apache.flink.streaming.api.function.source.SocketTextStreamFunction;
-import org.apache.flink.streaming.util.MockCollector;
-import org.apache.flink.streaming.util.MockSource;
-import org.junit.Test;
-
-public class SourceTest {
-
- @Test
- public void fromElementsTest() {
- List<Integer> expectedList = Arrays.asList(1, 2, 3);
- List<Integer> actualList = MockSource.createAndExecute(new FromElementsFunction<Integer>(1,
- 2, 3));
- assertEquals(expectedList, actualList);
- }
-
- @Test
- public void fromCollectionTest() {
- List<Integer> expectedList = Arrays.asList(1, 2, 3);
- List<Integer> actualList = MockSource.createAndExecute(new FromElementsFunction<Integer>(
- Arrays.asList(1, 2, 3)));
- assertEquals(expectedList, actualList);
- }
-
- @Test
- public void socketTextStreamTest() throws Exception {
- List<String> expectedList = Arrays.asList("a", "b", "c");
- List<String> actualList = new ArrayList<String>();
-
- byte[] data = { 'a', '\n', 'b', '\n', 'c', '\n' };
-
- Socket socket = mock(Socket.class);
- when(socket.getInputStream()).thenReturn(new ByteArrayInputStream(data));
- when(socket.isClosed()).thenReturn(false);
- when(socket.isConnected()).thenReturn(true);
-
- new SocketTextStreamFunction("", 0, '\n').streamFromSocket(new MockCollector<String>(
- actualList), socket);
- assertEquals(expectedList, actualList);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
deleted file mode 100644
index 3da6b5f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-
-import org.apache.flink.api.common.functions.CrossFunction;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.util.TestStreamEnvironment;
-import org.junit.Test;
-
-public class WindowCrossJoinTest implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private static final long MEMORYSIZE = 32;
-
- private static ArrayList<Tuple2<Tuple2<Integer, String>, Integer>> joinResults = new ArrayList<Tuple2<Tuple2<Integer, String>, Integer>>();
- private static ArrayList<Tuple2<Tuple2<Integer, String>, Integer>> joinExpectedResults = new ArrayList<Tuple2<Tuple2<Integer, String>, Integer>>();
-
- private static ArrayList<Tuple2<Tuple2<Integer, String>, Integer>> crossResults = new ArrayList<Tuple2<Tuple2<Integer, String>, Integer>>();
- private static ArrayList<Tuple2<Tuple2<Integer, String>, Integer>> crossExpectedResults = new ArrayList<Tuple2<Tuple2<Integer, String>, Integer>>();
-
- @Test
- public void test() throws Exception {
- StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
- env.setBufferTimeout(1);
-
- ArrayList<Tuple2<Integer, String>> in1 = new ArrayList<Tuple2<Integer, String>>();
- ArrayList<Tuple1<Integer>> in2 = new ArrayList<Tuple1<Integer>>();
-
- in1.add(new Tuple2<Integer, String>(10, "a"));
- in1.add(new Tuple2<Integer, String>(20, "b"));
- in1.add(new Tuple2<Integer, String>(20, "x"));
- in1.add(new Tuple2<Integer, String>(0, "y"));
-
- in2.add(new Tuple1<Integer>(0));
- in2.add(new Tuple1<Integer>(5));
- in2.add(new Tuple1<Integer>(20));
-
- joinExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(20, "b"), 20));
- joinExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(20, "x"), 20));
- joinExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(0, "y"), 0));
-
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(10, "a"), 0));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(10, "a"), 5));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(10, "a"), 20));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(20, "b"), 0));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(20, "b"), 5));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(20, "b"), 20));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(20, "x"), 0));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(20, "x"), 5));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(20, "x"), 20));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(0, "y"), 0));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(0, "y"), 5));
- crossExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
- new Tuple2<Integer, String>(0, "y"), 20));
-
- DataStream<Tuple2<Integer, String>> inStream1 = env.fromCollection(in1);
- DataStream<Tuple1<Integer>> inStream2 = env.fromCollection(in2);
-
- inStream1
- .join(inStream2)
- .onWindow(1000, new MyTimestamp<Tuple2<Integer, String>>(),
- new MyTimestamp<Tuple1<Integer>>(), 100).where(0).equalTo(0)
- .addSink(new JoinResultSink());
-
- inStream1
- .cross(inStream2)
- .onWindow(1000, new MyTimestamp<Tuple2<Integer, String>>(),
- new MyTimestamp<Tuple1<Integer>>(), 100)
- .with(new CrossFunction<Tuple2<Integer, String>, Tuple1<Integer>, Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<Tuple2<Integer, String>, Tuple1<Integer>> cross(
- Tuple2<Integer, String> val1, Tuple1<Integer> val2) throws Exception {
- return new Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>(val1, val2);
- }
- }).addSink(new CrossResultSink());
-
- env.execute();
-
- assertEquals(joinExpectedResults, joinResults);
- assertEquals(crossExpectedResults, crossResults);
- }
-
- private static class MyTimestamp<T> implements Timestamp<T> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(T value) {
- return 101L;
- }
- }
-
- private static class JoinResultSink implements
- SinkFunction<Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(Tuple2<Tuple2<Integer, String>, Tuple1<Integer>> value) {
- joinResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(value.f0, value.f1.f0));
- }
- }
-
- private static class CrossResultSink implements
- SinkFunction<Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(Tuple2<Tuple2<Integer, String>, Tuple1<Integer>> value) {
- crossResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(value.f0, value.f1.f0));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java
deleted file mode 100644
index 78cbbe5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/DirectedOutputTest.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import org.apache.flink.streaming.api.datastream.SplitDataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.util.TestStreamEnvironment;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-public class DirectedOutputTest {
-
- private static final String TEN = "ten";
- private static final String ODD = "odd";
- private static final String ALL = "all";
- private static final String EVEN_AND_ODD = "evenAndOdd";
- private static final String ODD_AND_TEN = "oddAndTen";
- private static final String EVEN = "even";
- private static final String NON_SELECTED = "nonSelected";
-
- static final class MyOutputSelector implements OutputSelector<Long> {
- private static final long serialVersionUID = 1L;
-
- List<String> outputs = new ArrayList<String>();
-
- @Override
- public Iterable<String> select(Long value) {
- outputs.clear();
- if (value % 2 == 0) {
- outputs.add(EVEN);
- } else {
- outputs.add(ODD);
- }
-
- if (value == 10L) {
- outputs.add(TEN);
- }
-
- if (value == 11L) {
- outputs.add(NON_SELECTED);
- }
- return outputs;
- }
- }
-
- static final class ListSink implements SinkFunction<Long> {
- private static final long serialVersionUID = 1L;
-
- private String name;
- private transient List<Long> list;
-
- public ListSink(String name) {
- this.name = name;
- }
-
- @Override
- public void invoke(Long value) {
- list.add(value);
- }
-
- private void readObject(java.io.ObjectInputStream in) throws IOException,
- ClassNotFoundException {
- in.defaultReadObject();
- outputs.put(name, new ArrayList<Long>());
- this.list = outputs.get(name);
- }
- }
-
- private static Map<String, List<Long>> outputs = new HashMap<String, List<Long>>();
-
- @Test
- public void outputSelectorTest() throws Exception {
- StreamExecutionEnvironment env = new TestStreamEnvironment(1, 128);
-
- SplitDataStream<Long> source = env.generateSequence(1, 11).split(new MyOutputSelector());
- source.select(EVEN).addSink(new ListSink(EVEN));
- source.select(ODD, TEN).addSink(new ListSink(ODD_AND_TEN));
- source.select(EVEN, ODD).addSink(new ListSink(EVEN_AND_ODD));
- source.addSink(new ListSink(ALL));
-
- env.execute();
- assertEquals(Arrays.asList(2L, 4L, 6L, 8L, 10L), outputs.get(EVEN));
- assertEquals(Arrays.asList(1L, 3L, 5L, 7L, 9L, 10L, 11L), outputs.get(ODD_AND_TEN));
- assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
- outputs.get(EVEN_AND_ODD));
- assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L), outputs.get(ALL));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java
deleted file mode 100755
index 1615a45..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/OutputSelectorTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.junit.Test;
-
-public class OutputSelectorTest {
-
- static final class MyOutputSelector implements OutputSelector<Tuple1<Integer>> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Iterable<String> select(Tuple1<Integer> tuple) {
-
- String[] outputs = new String[tuple.f0];
-
- for (Integer i = 0; i < tuple.f0; i++) {
- outputs[i] = i.toString();
- }
- return Arrays.asList(outputs);
- }
- }
-
- @Test
- public void testGetOutputs() {
- OutputSelector<Tuple1<Integer>> selector = new MyOutputSelector();
- List<String> expectedOutputs = new ArrayList<String>();
- expectedOutputs.add("0");
- expectedOutputs.add("1");
- assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(2)));
- expectedOutputs.add("2");
- assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(3)));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
deleted file mode 100755
index 49b3bf8..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import static org.junit.Assert.assertArrayEquals;
-
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.streamvertex.MockRecordWriter;
-import org.apache.flink.streaming.util.MockRecordWriterFactory;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class StreamCollectorTest {
-
- @Test
- public void testCollect() {
- MockRecordWriter recWriter = MockRecordWriterFactory.create();
- SerializationDelegate<StreamRecord<Tuple1<Integer>>> sd = new SerializationDelegate<StreamRecord<Tuple1<Integer>>>(
- null);
- sd.setInstance(new StreamRecord<Tuple1<Integer>>().setObject(new Tuple1<Integer>()));
-
- Collector<Tuple1<Integer>> collector = new StreamOutput<Tuple1<Integer>>(recWriter, 2, sd);
- collector.collect(new Tuple1<Integer>(3));
- collector.collect(new Tuple1<Integer>(4));
- collector.collect(new Tuple1<Integer>(5));
- collector.collect(new Tuple1<Integer>(6));
-
- assertArrayEquals(new Integer[] { 3, 4, 5, 6 }, recWriter.emittedRecords.toArray());
- }
-
- @Test
- public void testClose() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java
deleted file mode 100644
index a91bd0c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoFlatMapInvokable;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class CoFlatMapTest implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private final static class MyCoFlatMap implements CoFlatMapFunction<String, Integer, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap1(String value, Collector<String> coll) {
- for (int i = 0; i < value.length(); i++) {
- coll.collect(value.substring(i, i + 1));
- }
- }
-
- @Override
- public void flatMap2(Integer value, Collector<String> coll) {
- coll.collect(value.toString());
- }
- }
-
- @Test
- public void coFlatMapTest() {
- CoFlatMapInvokable<String, Integer, String> invokable = new CoFlatMapInvokable<String, Integer, String>(
- new MyCoFlatMap());
-
- List<String> expectedList = Arrays.asList("a", "b", "c", "1", "d", "e", "f", "2", "g", "h",
- "e", "3", "4", "5");
- List<String> actualList = MockCoContext.createAndExecute(invokable,
- Arrays.asList("abc", "def", "ghe"), Arrays.asList(1, 2, 3, 4, 5));
-
- assertEquals(expectedList, actualList);
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void multipleInputTest() {
- LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- DataStream<Integer> ds1 = env.fromElements(1, 3, 5);
- DataStream<Integer> ds2 = env.fromElements(2, 4).merge(ds1);
-
- try {
- ds1.forward().merge(ds2);
- fail();
- } catch (RuntimeException e) {
- // expected
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
deleted file mode 100644
index 273bbae..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedReduceInvokable;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.junit.Test;
-
-public class CoGroupedReduceTest {
-
- private final static class MyCoReduceFunction implements
- CoReduceFunction<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple3<String, String, String> reduce1(Tuple3<String, String, String> value1,
- Tuple3<String, String, String> value2) {
- return new Tuple3<String, String, String>(value1.f0, value1.f1 + value2.f1, value1.f2);
- }
-
- @Override
- public Tuple2<Integer, Integer> reduce2(Tuple2<Integer, Integer> value1,
- Tuple2<Integer, Integer> value2) {
- return new Tuple2<Integer, Integer>(value1.f0, value1.f1 + value2.f1);
- }
-
- @Override
- public String map1(Tuple3<String, String, String> value) {
- return value.f1;
- }
-
- @Override
- public String map2(Tuple2<Integer, Integer> value) {
- return value.f1.toString();
- }
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void coGroupedReduceTest() {
- Tuple3<String, String, String> word1 = new Tuple3<String, String, String>("a", "word1", "b");
- Tuple3<String, String, String> word2 = new Tuple3<String, String, String>("b", "word2", "a");
- Tuple3<String, String, String> word3 = new Tuple3<String, String, String>("a", "word3", "a");
- Tuple2<Integer, Integer> int1 = new Tuple2<Integer, Integer>(2, 1);
- Tuple2<Integer, Integer> int2 = new Tuple2<Integer, Integer>(1, 2);
- Tuple2<Integer, Integer> int3 = new Tuple2<Integer, Integer>(0, 3);
- Tuple2<Integer, Integer> int4 = new Tuple2<Integer, Integer>(2, 4);
- Tuple2<Integer, Integer> int5 = new Tuple2<Integer, Integer>(1, 5);
-
- KeySelector<Tuple3<String, String, String>, ?> keySelector0 = new KeySelector<Tuple3<String, String, String>, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String getKey(Tuple3<String, String, String> value) throws Exception {
- return value.f0;
- }
- };
-
- KeySelector<Tuple2<Integer, Integer>, ?> keySelector1 = new KeySelector<Tuple2<Integer, Integer>, Integer>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(Tuple2<Integer, Integer> value) throws Exception {
- return value.f0;
- }
- };
-
- KeySelector<Tuple3<String, String, String>, ?> keySelector2 = new KeySelector<Tuple3<String, String, String>, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String getKey(Tuple3<String, String, String> value) throws Exception {
- return value.f2;
- }
- };
-
- CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> invokable = new CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
- new MyCoReduceFunction(), keySelector0, keySelector1);
-
- List<String> expected = Arrays.asList("word1", "1", "word2", "2", "word1word3", "3", "5",
- "7");
-
- List<String> actualList = MockCoContext.createAndExecute(invokable,
- Arrays.asList(word1, word2, word3), Arrays.asList(int1, int2, int3, int4, int5));
-
- assertEquals(expected, actualList);
-
- invokable = new CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
- new MyCoReduceFunction(), keySelector2, keySelector1);
-
- expected = Arrays.asList("word1", "1", "word2", "2", "word2word3", "3", "5", "7");
-
- actualList = MockCoContext.createAndExecute(invokable, Arrays.asList(word1, word2, word3),
- Arrays.asList(int1, int2, int3, int4, int5));
-
- assertEquals(expected, actualList);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java
deleted file mode 100644
index 93d1741..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.api.function.co.CoMapFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoMapInvokable;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.junit.Test;
-
-public class CoMapTest implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private final static class MyCoMap implements CoMapFunction<Double, Integer, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public String map1(Double value) {
- return value.toString();
- }
-
- @Override
- public String map2(Integer value) {
- return value.toString();
- }
- }
-
- @Test
- public void coMapTest() {
- CoMapInvokable<Double, Integer, String> invokable = new CoMapInvokable<Double, Integer, String>(new MyCoMap());
-
- List<String> expectedList = Arrays.asList("1.1", "1", "1.2", "2", "1.3", "3", "1.4", "1.5");
- List<String> actualList = MockCoContext.createAndExecute(invokable, Arrays.asList(1.1, 1.2, 1.3, 1.4, 1.5), Arrays.asList(1, 2, 3));
-
- assertEquals(expectedList, actualList);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java
deleted file mode 100755
index 3343ba0..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoReduceInvokable;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.junit.Test;
-
-public class CoStreamReduceTest {
-
- public static class MyCoReduceFunction implements
- CoReduceFunction<Integer, String, Integer> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce1(Integer value1, Integer value2) {
- return value1 * value2;
- }
-
- @Override
- public String reduce2(String value1, String value2) {
- return value1 + value2;
- }
-
- @Override
- public Integer map1(Integer value) {
- return value;
- }
-
- @Override
- public Integer map2(String value) {
- return Integer.parseInt(value);
- }
-
- }
-
- @Test
- public void coStreamReduceTest() {
-
- CoReduceInvokable<Integer, String, Integer> coReduce = new CoReduceInvokable<Integer, String, Integer>(
- new MyCoReduceFunction());
-
- List<Integer> expected1 = Arrays.asList(1, 9, 2, 99, 6, 998, 24);
- List<Integer> result = MockCoContext.createAndExecute(coReduce,
- Arrays.asList(1, 2, 3, 4), Arrays.asList("9", "9", "8"));
-
- assertEquals(expected1, result);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
deleted file mode 100644
index 4ab3492..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.function.co.CoWindowFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class CoWindowTest {
-
- public static final class MyCoGroup1 implements CoWindowFunction<Integer, Integer, Integer> {
-
- private static final long serialVersionUID = 1L;
-
- @SuppressWarnings("unused")
- @Override
- public void coWindow(List<Integer> first, List<Integer> second, Collector<Integer> out)
- throws Exception {
- Integer count1 = 0;
- for (Integer i : first) {
- count1++;
- }
- Integer count2 = 0;
- for (Integer i : second) {
- count2++;
- }
- out.collect(count1);
- out.collect(count2);
-
- }
-
- }
-
- public static final class MyCoGroup2 implements
- CoWindowFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void coWindow(List<Tuple2<Integer, Integer>> first,
- List<Tuple2<Integer, Integer>> second, Collector<Integer> out) throws Exception {
-
- Set<Integer> firstElements = new HashSet<Integer>();
- for (Tuple2<Integer, Integer> value : first) {
- firstElements.add(value.f1);
- }
- for (Tuple2<Integer, Integer> value : second) {
- if (firstElements.contains(value.f1)) {
- out.collect(value.f1);
- }
- }
-
- }
-
- }
-
- private static final class MyTS1 implements Timestamp<Integer> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Integer value) {
- return value;
- }
-
- }
-
- private static final class MyTS2 implements Timestamp<Tuple2<Integer, Integer>> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Tuple2<Integer, Integer> value) {
- return value.f0;
- }
-
- }
-
- @Test
- public void coWindowGroupReduceTest2() throws Exception {
-
- CoWindowInvokable<Integer, Integer, Integer> invokable1 = new CoWindowInvokable<Integer, Integer, Integer>(
- new MyCoGroup1(), 2, 1, new TimestampWrapper<Integer>(new MyTS1(), 1),
- new TimestampWrapper<Integer>(new MyTS1(), 1));
-
- // Windowsize 2, slide 1
- // 1,2|2,3|3,4|4,5
-
- List<Integer> input11 = new ArrayList<Integer>();
- input11.add(1);
- input11.add(1);
- input11.add(2);
- input11.add(3);
- input11.add(3);
-
- List<Integer> input12 = new ArrayList<Integer>();
- input12.add(1);
- input12.add(2);
- input12.add(3);
- input12.add(3);
- input12.add(5);
-
- // Windows: (1,1,2)(1,1,2)|(2,3,3)(2,3,3)|(3,3)(3,3)|(5)(5)
- // expected output: 3,2|3,3|2,2|0,1
-
- List<Integer> expected1 = new ArrayList<Integer>();
- expected1.add(3);
- expected1.add(2);
- expected1.add(3);
- expected1.add(3);
- expected1.add(2);
- expected1.add(2);
- expected1.add(0);
- expected1.add(1);
-
- List<Integer> actual1 = MockCoContext.createAndExecute(invokable1, input11, input12);
- assertEquals(expected1, actual1);
-
- CoWindowInvokable<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> invokable2 = new CoWindowInvokable<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer>(
- new MyCoGroup2(), 2, 3, new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(),
- 1), new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(), 1));
-
- // WindowSize 2, slide 3
- // 1,2|4,5|7,8|
-
- List<Tuple2<Integer, Integer>> input21 = new ArrayList<Tuple2<Integer, Integer>>();
- input21.add(new Tuple2<Integer, Integer>(1, 1));
- input21.add(new Tuple2<Integer, Integer>(1, 2));
- input21.add(new Tuple2<Integer, Integer>(2, 3));
- input21.add(new Tuple2<Integer, Integer>(3, 4));
- input21.add(new Tuple2<Integer, Integer>(3, 5));
- input21.add(new Tuple2<Integer, Integer>(4, 6));
- input21.add(new Tuple2<Integer, Integer>(4, 7));
- input21.add(new Tuple2<Integer, Integer>(5, 8));
-
- List<Tuple2<Integer, Integer>> input22 = new ArrayList<Tuple2<Integer, Integer>>();
- input22.add(new Tuple2<Integer, Integer>(1, 1));
- input22.add(new Tuple2<Integer, Integer>(2, 0));
- input22.add(new Tuple2<Integer, Integer>(2, 2));
- input22.add(new Tuple2<Integer, Integer>(3, 9));
- input22.add(new Tuple2<Integer, Integer>(3, 4));
- input22.add(new Tuple2<Integer, Integer>(4, 10));
- input22.add(new Tuple2<Integer, Integer>(5, 8));
- input22.add(new Tuple2<Integer, Integer>(5, 7));
-
- List<Integer> expected2 = new ArrayList<Integer>();
- expected2.add(1);
- expected2.add(2);
- expected2.add(8);
- expected2.add(7);
-
- List<Integer> actual2 = MockCoContext.createAndExecute(invokable2, input21, input22);
- assertEquals(expected2, actual2);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokableTest.java
deleted file mode 100644
index 969a06b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokableTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class CounterInvokableTest {
-
- @Test
- public void counterTest() {
- CounterInvokable<String> invokable = new CounterInvokable<String>();
-
- List<Long> expected = Arrays.asList(1L, 2L, 3L);
- List<Long> actual = MockContext.createAndExecute(invokable, Arrays.asList("one", "two", "three"));
-
- assertEquals(expected, actual);
- }
-}
[35/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java
deleted file mode 100644
index 9d055d5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.deltafunction;
-
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
-
-/**
- * This delta function calculates the euclidean distance between two given
- * points.
- *
- * Euclidean distance: http://en.wikipedia.org/wiki/Euclidean_distance
- *
- * @param <DATA>
- * The input data type. This delta function works with a double[],
- * but can extract/convert to it from any other given object in case
- * the respective extractor has been set. See
- * {@link ExtractionAwareDeltaFunction} for more information.
- */
-public class EuclideanDistance<DATA> extends ExtractionAwareDeltaFunction<DATA, double[]> {
-
- public EuclideanDistance() {
- super(null);
- }
-
- public EuclideanDistance(Extractor<DATA, double[]> converter) {
- super(converter);
- }
-
- /**
- * auto-generated version id
- */
- private static final long serialVersionUID = 3119432599634512359L;
-
- @Override
- public double getNestedDelta(double[] oldDataPoint, double[] newDataPoint) {
- double result = 0;
- for (int i = 0; i < oldDataPoint.length; i++) {
- result += (oldDataPoint[i] - newDataPoint[i]) * (oldDataPoint[i] - newDataPoint[i]);
- }
- return Math.sqrt(result);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java
deleted file mode 100644
index 81d01a4..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.deltafunction;
-
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
-
-/**
- * Extend this abstract class to implement a delta function which is aware of
- * extracting the data on which the delta is calculated from a more complex data
- * structure. For example in case you want to be able to run a delta only on one
- * field of a Tuple type or only on some fields from an array.
- *
- * @param <DATA>
- * The input data type. The input of this type will be passed to the
- * extractor which will transform into a TO-object. The delta
- * function then runs on this TO-object.
- * @param <TO>
- * The type on which the delta function runs. (The type of the delta
- * function)
- */
-public abstract class ExtractionAwareDeltaFunction<DATA, TO> implements DeltaFunction<DATA> {
-
- /**
- * Generated Version ID
- */
- private static final long serialVersionUID = 6927486219702689554L;
- private Extractor<DATA, TO> converter;
-
- public ExtractionAwareDeltaFunction(Extractor<DATA, TO> converter) {
- this.converter = converter;
- }
-
- /**
- * This method takes the two data point and runs the set extractor on it.
- * The delta function implemented at {@link getNestedDelta} is then called
- * with the extracted data. In case no extractor is set the input data gets
- * passes to {@link getNestedDelta} as-is. The return value is just
- * forwarded from {@link getNestedDelta}.
- *
- * @param oldDataPoint
- * the older data point as raw data (before extraction).
- * @param newDataPoint
- * the new data point as raw data (before extraction).
- * @return the delta between the two points.
- */
- @SuppressWarnings("unchecked")
- @Override
- public double getDelta(DATA oldDataPoint, DATA newDataPoint) {
- if (converter == null) {
- // In case no conversion/extraction is required, we can cast DATA to
- // TO
- // => Therefore, "unchecked" warning is suppressed for this method.
- return getNestedDelta((TO) oldDataPoint, (TO) newDataPoint);
- } else {
- return getNestedDelta(converter.extract(oldDataPoint), converter.extract(newDataPoint));
- }
-
- }
-
- /**
- * This method is exactly the same as
- * {@link DeltaFunction#getDelta(Object, Object)} except that it gets the
- * result of the previously done extractions as input. Therefore, this
- * method only does the actual calculation of the delta but no data
- * extraction or conversion.
- *
- * @param oldDataPoint
- * the older data point.
- * @param newDataPoint
- * the new data point.
- * @return the delta between the two points.
- */
- public abstract double getNestedDelta(TO oldDataPoint, TO newDataPoint);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java
deleted file mode 100644
index ee878ac..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import org.apache.flink.api.java.tuple.Tuple;
-
-/**
- * Converts a Tuple to an Object-Array. The field which should be included in
- * the array can selected and reordered as needed.
- */
-public class ArrayFromTuple implements Extractor<Tuple, Object[]> {
-
- /**
- * Auto generated version id
- */
- private static final long serialVersionUID = -6076121226427616818L;
- int[] order = null;
-
- /**
- * Using this constructor the extractor will convert the whole tuple (all
- * fields in the original order) to an array.
- */
- public ArrayFromTuple() {
- // noting to do
- }
-
- /**
- * Using this constructor the extractor will combine the fields as specified
- * in the indexes parameter in an object array.
- *
- * @param indexes
- * the field ids (enumerated from 0)
- */
- public ArrayFromTuple(int... indexes) {
- this.order = indexes;
- }
-
- @Override
- public Object[] extract(Tuple in) {
- Object[] output;
-
- if (order == null) {
- // copy the whole tuple
- output = new Object[in.getArity()];
- for (int i = 0; i < in.getArity(); i++) {
- output[i] = in.getField(i);
- }
- } else {
- // copy user specified order
- output = new Object[order.length];
- for (int i = 0; i < order.length; i++) {
- output[i] = in.getField(order[i]);
- }
- }
-
- return output;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java
deleted file mode 100644
index 9048a3c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-/**
- * Combines two extractors which will be executed one after each other.
- *
- * @param <FROM>
- * The input type of the first extractor.
- * @param <OVER>
- * The output type of the first and the input type of the second
- * extractor.
- * @param <TO>
- * The output type of the second extractor and the output type of the
- * over all extraction.
- */
-public class ConcatinatedExtract<FROM, OVER, TO> implements Extractor<FROM, TO> {
-
- /**
- * auto-generated id
- */
- private static final long serialVersionUID = -7807197760725651752L;
-
- private Extractor<FROM, OVER> e1;
- private Extractor<OVER, TO> e2;
-
- /**
- * Combines two extractors which will be executed one after each other.
- *
- * @param e1
- * First extractor: This extractor gets applied to the input data
- * first. Its output as then passed as input to the second
- * extractor.
- * @param e2
- * Second extractor: This extractor gets the output of the first
- * extractor as input. Its output is then the result of the over
- * all extraction.
- */
- public ConcatinatedExtract(Extractor<FROM, OVER> e1, Extractor<OVER, TO> e2) {
- this.e1 = e1;
- this.e2 = e2;
- }
-
- @Override
- public TO extract(FROM in) {
- return e2.extract(e1.extract(in));
- }
-
- public <OUT> ConcatinatedExtract<FROM, TO, OUT> add(Extractor<TO, OUT> e3) {
- return new ConcatinatedExtract<FROM, TO, OUT>(this, e3);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java
deleted file mode 100644
index b103ca3..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import java.io.Serializable;
-
-/**
- * Extractors allow to extract/convert one type to another. They are mostly used
- * to extract some fields out of a more complex structure (Tuple/Array) to run
- * further calculation on the extraction result.
- *
- * @param <FROM>
- * The input data type.
- * @param <TO>
- * The output data type.
- */
-public interface Extractor<FROM, TO> extends Serializable {
-
- /**
- * Extracts/Converts the given input to an object of the output type
- *
- * @param in
- * the input data
- * @return the extracted/converted data
- */
- public TO extract(FROM in);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java
deleted file mode 100644
index 0568276..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import java.lang.reflect.Array;
-
-/**
- * Extracts a single field out of an array.
- *
- * @param <OUT>
- * The type of the extracted field.
- */
-public class FieldFromArray<OUT> implements Extractor<Object, OUT> {
-
- /**
- * Auto-gernated version id
- */
- private static final long serialVersionUID = -5161386546695574359L;
- private int fieldId = 0;
-
- /**
- * Extracts the first field (id 0) from the array
- */
- public FieldFromArray() {
- // noting to do => will use default 0
- }
-
- /**
- * Extracts the field with the given id from the array.
- *
- * @param fieldId
- * The id of the field which will be extracted from the array.
- */
- public FieldFromArray(int fieldId) {
- this.fieldId = fieldId;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public OUT extract(Object in) {
- return (OUT) Array.get(in, fieldId);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java
deleted file mode 100644
index 07b38f5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import org.apache.flink.api.java.tuple.Tuple;
-
-/**
- * Extracts a single field out of a tuple.
- *
- * @param <OUT>
- * The type of the extracted field.
- */
-public class FieldFromTuple<OUT> implements Extractor<Tuple, OUT> {
-
- /**
- * Auto-gernated version id
- */
- private static final long serialVersionUID = -5161386546695574359L;
- private int fieldId = 0;
-
- /**
- * Extracts the first field (id 0) from the tuple
- */
- public FieldFromTuple() {
- // noting to do => will use default 0
- }
-
- /**
- * Extracts the field with the given id from the tuple.
- *
- * @param fieldId
- * The id of the field which will be extracted from the tuple.
- */
- public FieldFromTuple(int fieldId) {
- this.fieldId = fieldId;
- }
-
- @Override
- public OUT extract(Tuple in) {
- return in.getField(fieldId);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java
deleted file mode 100644
index 4e98689..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import java.lang.reflect.Array;
-
-/**
- * Extracts multiple fields from an array and puts them into a new array of the
- * specified type.
- *
- * @param <OUT>
- * The type of the output array. If out is set to String, the output
- * of the extractor will be a String[]. If it is set to String[] the
- * output will be String[][].
- */
-public class FieldsFromArray<OUT> implements Extractor<Object, OUT[]> {
-
- /**
- * Auto-generated version id
- */
- private static final long serialVersionUID = 8075055384516397670L;
- private int[] order;
- private Class<OUT> clazz;
-
- /**
- * Extracts multiple fields from an array and puts them in the given order
- * into a new array of the specified type.
- *
- * @param clazz
- * the Class object representing the component type of the new
- * array
- * @param indexes
- * The indexes of the fields to be extracted. Any order is
- * possible, but not more than 255 fields due to limitations in
- * {@link Array#newInstance(Class, int...)}.
- */
- public FieldsFromArray(Class<OUT> clazz, int... indexes) {
- this.order = indexes;
- this.clazz = clazz;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public OUT[] extract(Object in) {
- OUT[] output = (OUT[]) Array.newInstance(clazz, order.length);
- for (int i = 0; i < order.length; i++) {
- output[i] = (OUT) Array.get(in, this.order[i]);
- }
- return output;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
deleted file mode 100644
index 1bfc461..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.extractor;
-
-import org.apache.flink.api.java.tuple.Tuple;
-
-/**
- * Extracts one or more fields of the type Double from a tuple and puts them
- * into a new double[]
- */
-public class FieldsFromTuple implements Extractor<Tuple, double[]> {
-
- /**
- * auto generated version id
- */
- private static final long serialVersionUID = -2554079091050273761L;
- int[] indexes;
-
- /**
- * Extracts one or more fields of the the type Double from a tuple and puts
- * them into a new double[] (in the specified order).
- *
- * @param indexes
- * The indexes of the fields to be extracted.
- */
- public FieldsFromTuple(int... indexes) {
- this.indexes = indexes;
- }
-
- @Override
- public double[] extract(Tuple in) {
- double[] out = new double[indexes.length];
- for (int i = 0; i < indexes.length; i++) {
- out[i] = (Double) in.getField(indexes[i]);
- }
- return out;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
deleted file mode 100644
index 840546f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.helper;
-
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-/**
- * Represents a count based trigger or eviction policy. Use the
- * {@link Count#of(int)} to get an instance.
- */
-@SuppressWarnings("rawtypes")
-public class Count implements WindowingHelper {
-
- private int count;
- private int deleteOnEviction = 1;
- private int startValue = CountTriggerPolicy.DEFAULT_START_VALUE;
-
- /**
- * Specifies on which element a trigger or an eviction should happen (based
- * on the count of the elements).
- *
- * This constructor does exactly the same as {@link Count#of(int)}.
- *
- * @param count
- * the number of elements to count before trigger/evict
- */
- public Count(int count) {
- this.count = count;
- }
-
- @Override
- public EvictionPolicy<?> toEvict() {
- return new CountEvictionPolicy(count, deleteOnEviction);
- }
-
- @Override
- public TriggerPolicy<?> toTrigger() {
- return new CountTriggerPolicy(count, startValue);
- }
-
- /**
- * Sets the number of elements deleted at each eviction (i.e when the number
- * elements exceeds the window size). By default the elements get deleted
- * one by one (deleteOnEvition = 1)
- *
- * @param deleteOnEviction
- * The number of elements deleted at each evition
- * @return Helper representing the count based policy
- *
- */
- public Count withDelete(int deleteOnEviction) {
- this.deleteOnEviction = deleteOnEviction;
- return this;
- }
-
- /**
- * Sets the initial value of the counter. 0 by default
- *
- * @param startValue
- * Starting value of the window counter
- * @return Helper representing the count based policy
- *
- */
- public Count startingAt(int startValue) {
- this.startValue = startValue;
- return this;
- }
-
- /**
- * Specifies a count based eviction (window size) or trigger policy (slide
- * size). For eviction 'count' defines the number of elements in each
- * window. For trigger 'count' defines how often do we call the user
- * function in terms of number of elements received.
- *
- * @param count
- * the number of elements to count before trigger/evict
- * @return Helper representing the count based policy
- */
- public static Count of(int count) {
- return new Count(count);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
deleted file mode 100644
index 5434a4e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.helper;
-
-import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
-import org.apache.flink.streaming.api.windowing.policy.DeltaPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-/**
- * This helper represents a trigger or eviction policy based on a
- * {@link DeltaFunction}.
- *
- * @param <DATA>
- * the data type handled by the delta function represented by this
- * helper.
- */
-public class Delta<DATA> implements WindowingHelper<DATA> {
-
- private DeltaFunction<DATA> deltaFunction;
- private DATA initVal;
- private double threshold;
-
- /**
- * Creates a delta helper representing a delta count or eviction policy
- *
- * @param deltaFunction
- * The delta function which should be used to calculate the delta
- * between points.
- * @param initVal
- * The initial value which will be used to calculate the first
- * delta.
- * @param threshold
- * The threshold used by the delta function.
- */
- public Delta(DeltaFunction<DATA> deltaFunction, DATA initVal, double threshold) {
- this.deltaFunction = deltaFunction;
- this.initVal = initVal;
- this.threshold = threshold;
- }
-
- @Override
- public EvictionPolicy<DATA> toEvict() {
- return new DeltaPolicy<DATA>(deltaFunction, initVal, threshold);
- }
-
- @Override
- public TriggerPolicy<DATA> toTrigger() {
- return new DeltaPolicy<DATA>(deltaFunction, initVal, threshold);
- }
-
- /**
- * Creates a delta helper representing a delta trigger or eviction policy.
- * </br></br> This policy calculates a delta between the data point which
- * triggered last and the currently arrived data point. It triggers if the
- * delta is higher than a specified threshold. </br></br> In case it gets
- * used for eviction, this policy starts from the first element of the
- * buffer and removes all elements from the buffer which have a higher delta
- * then the threshold. As soon as there is an element with a lower delta,
- * the eviction stops.
- *
- * @param deltaFunction
- * The delta function which should be used to calculate the delta
- * between points.
- * @param initVal
- * The initial value which will be used to calculate the first
- * delta.
- * @param threshold
- * The threshold used by the delta function.
- * @return Helper representing a delta trigger or eviction policy
- */
- public static <DATA> Delta<DATA> of(double threshold, DeltaFunction<DATA> deltaFunction,
- DATA initVal) {
- return new Delta<DATA>(deltaFunction, initVal, threshold);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java
deleted file mode 100644
index 8581ac5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.helper;
-
-/**
- * {@link Timestamp} implementation to be used when system time is needed to
- * determine windows
- */
-public class SystemTimestamp<T> implements Timestamp<T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(T value) {
- return System.currentTimeMillis();
- }
-
- public static <R> TimestampWrapper<R> getWrapper() {
- return new TimestampWrapper<R>(new SystemTimestamp<R>(), System.currentTimeMillis());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
deleted file mode 100644
index f94eea4..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.helper;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-/**
- * This helper represents a time based count or eviction policy. By default the
- * time is measured with {@link System#currentTimeMillis()} in
- * {@link DefaultTimeStamp}.
- *
- * @param <DATA>
- * The data type which is handled by the time stamp used in the
- * policy represented by this helper
- */
-public class Time<DATA> implements WindowingHelper<DATA> {
-
- protected long length;
- protected TimeUnit granularity;
- protected TimestampWrapper<DATA> timestampWrapper;
- protected long delay;
-
- /**
- * Creates a helper representing a trigger which triggers every given
- * length or an eviction which evicts all elements older than length.
- *
- * @param length
- * The number of time units
- * @param timeUnit
- * The unit of time such as minute oder millisecond. Note that
- * the smallest possible granularity is milliseconds. Any smaller
- * time unit might cause an error at runtime due to conversion
- * problems.
- * @param timestamp
- * The user defined timestamp that will be used to extract time
- * information from the incoming elements
- * @param startTime
- * The startTime of the stream for computing the first window
- */
- private Time(long length, TimeUnit timeUnit, Timestamp<DATA> timestamp, long startTime) {
- this(length, timeUnit, new TimestampWrapper<DATA>(timestamp, startTime));
- }
-
- /**
- * Creates a helper representing a trigger which triggers every given
- * length or an eviction which evicts all elements older than length.
- *
- * @param length
- * The number of time units
- * @param timeUnit
- * The unit of time such as minute oder millisecond. Note that
- * the smallest possible granularity is milliseconds. Any smaller
- * time unit might cause an error at runtime due to conversion
- * problems.
- * @param timestampWrapper
- * The user defined {@link TimestampWrapper} that will be used to
- * extract time information from the incoming elements
- */
- private Time(long length, TimeUnit timeUnit, TimestampWrapper<DATA> timestampWrapper) {
- this.length = length;
- this.granularity = timeUnit;
- this.timestampWrapper = timestampWrapper;
- this.delay = 0;
- }
-
- @Override
- public EvictionPolicy<DATA> toEvict() {
- return new TimeEvictionPolicy<DATA>(granularityInMillis(), timestampWrapper);
- }
-
- @Override
- public TriggerPolicy<DATA> toTrigger() {
- return new TimeTriggerPolicy<DATA>(granularityInMillis(), timestampWrapper, delay);
- }
-
- /**
- * Creates a helper representing a time trigger which triggers every given
- * length (slide size) or a time eviction which evicts all elements older
- * than length (window size) using System time.
- *
- * @param length
- * The number of time units
- * @param timeUnit
- * The unit of time such as minute oder millisecond. Note that
- * the smallest possible granularity is milliseconds. Any smaller
- * time unit might cause an error at runtime due to conversion
- * problems.
- * @return Helper representing the time based trigger and eviction policy
- */
- @SuppressWarnings("unchecked")
- public static <DATA> Time<DATA> of(long length, TimeUnit timeUnit) {
- return new Time<DATA>(length, timeUnit,
- (TimestampWrapper<DATA>) SystemTimestamp.getWrapper());
- }
-
- /**
- * Creates a helper representing a time trigger which triggers every given
- * length (slide size) or a time eviction which evicts all elements older
- * than length (window size) using a user defined timestamp extractor.
- *
- * @param length
- * The number of time units
- * @param timestamp
- * The user defined timestamp that will be used to extract time
- * information from the incoming elements
- * @param startTime
- * The startTime used to compute the first window
- * @return Helper representing the time based trigger and eviction policy
- */
- public static <DATA> Time<DATA> of(long length, Timestamp<DATA> timestamp, long startTime) {
- return new Time<DATA>(length, null, timestamp, startTime);
- }
-
- /**
- * Creates a helper representing a time trigger which triggers every given
- * length (slide size) or a time eviction which evicts all elements older
- * than length (window size) using a user defined timestamp extractor. By
- * default the start time is set to 0.
- *
- * @param length
- * The number of time units
- * @param timestamp
- * The user defined timestamp that will be used to extract time
- * information from the incoming elements
- * @return Helper representing the time based trigger and eviction policy
- */
- public static <DATA> Time<DATA> of(long length, Timestamp<DATA> timestamp) {
- return of(length, timestamp, 0);
- }
-
- /**
- * Sets the delay for the first processed window.
- *
- * @param delay
- * The number of time units before the first processed window.
- * @return Helper representing the time based trigger and eviction policy
- */
- public Time<DATA> withDelay(long delay) {
- this.delay = delay;
- return this;
- }
-
- protected long granularityInMillis() {
- return granularity == null ? length : granularity.toMillis(length);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java
deleted file mode 100644
index fea6020..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.helper;
-
-import java.io.Serializable;
-
-/**
- * Interface for getting a timestamp from a custom value. Used in window
- * reduces. In order to work properly, the timestamps must be non-decreasing.
- *
- * @param <T>
- * Type of the value to create the timestamp from.
- */
-public interface Timestamp<T> extends Serializable {
-
- /**
- * Values
- *
- * @param value
- * The value to create the timestamp from
- * @return The timestamp
- */
- public long getTimestamp(T value);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java
deleted file mode 100644
index 8c3a09d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.helper;
-
-import java.io.Serializable;
-
-public class TimestampWrapper<T> implements Serializable {
-
- private static final long serialVersionUID = 1L;
- private long startTime;
- private Timestamp<T> timestamp;
-
- public TimestampWrapper(Timestamp<T> timeStamp, long startTime) {
- this.timestamp = timeStamp;
- this.startTime = startTime;
- }
-
- public long getTimestamp(T in) {
- return timestamp.getTimestamp(in);
- }
-
- public long getStartTime() {
- return startTime;
- }
-
- public boolean isDefaultTimestamp() {
- return timestamp instanceof SystemTimestamp;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java
deleted file mode 100644
index 9df8432..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.helper;
-
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-/**
- * A helper representing a count or eviction policy. Such helper classes are
- * used to provide a nice and well readable API.
- *
- * @param <DATA>
- * the type of input data handled by this helper
- * @see Count
- * @see Time
- * @see Delta
- */
-public interface WindowingHelper<DATA> {
-
- public EvictionPolicy<DATA> toEvict();
-
- public TriggerPolicy<DATA> toTrigger();
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java
deleted file mode 100644
index 29ba9eb..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-/**
- * The {@link ActiveEvictionPolicy} wraps around a non active
- * {@link EvictionPolicy}. It forwards all calls to
- * {@link ActiveEvictionPolicy#notifyEvictionWithFakeElement(Object, int)} to
- * {@link EvictionPolicy#notifyEviction(Object, boolean, int)} while the
- * triggered parameter will be set to true.
- *
- * This class additionally implements the clone method and can wrap around
- * {@link CloneableEvictionPolicy} to make it active.
- *
- * @param <DATA>
- * The data type handled by this policy
- */
-public class ActiveCloneableEvictionPolicyWrapper<DATA> extends ActiveEvictionPolicyWrapper<DATA>
- implements CloneableEvictionPolicy<DATA> {
-
- /**
- * Auto generated version ID
- */
- private static final long serialVersionUID = 1520261575300622769L;
- CloneableEvictionPolicy<DATA> nestedPolicy;
-
- /**
- * Creates a wrapper which activates the eviction policy which is wrapped
- * in. This means that the nested policy will get called on fake elements as
- * well as on real elements.
- *
- * This specialized version of the {@link ActiveEvictionPolicyWrapper} works
- * with {@link CloneableEvictionPolicy} and is thereby cloneable as well.
- *
- * @param nestedPolicy
- * The policy which should be activated/wrapped in.
- */
- public ActiveCloneableEvictionPolicyWrapper(CloneableEvictionPolicy<DATA> nestedPolicy) {
- super(nestedPolicy);
- this.nestedPolicy = nestedPolicy;
- }
-
- @Override
- public ActiveCloneableEvictionPolicyWrapper<DATA> clone() {
- return new ActiveCloneableEvictionPolicyWrapper<DATA>(nestedPolicy.clone());
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
deleted file mode 100644
index fe172bc..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-/**
- * This interface is used for active eviction policies. beside the functionality
- * inherited from {@link EvictionPolicy} it provides a method which gets called
- * to notify on fake elements.
- *
- * In case an eviction policy implements this interface instead of the
- * {@link EvictionPolicy} interface, not only the real but also fake data points
- * will cause a notification of the eviction.
- *
- * Fake data points are mostly used in windowing based on time to trigger and
- * evict even if no element arrives at all during a windows duration.
- */
-public interface ActiveEvictionPolicy<DATA> extends EvictionPolicy<DATA> {
-
- /**
- * Proves if and how many elements should be deleted from the element
- * buffer. The eviction takes place after the trigger and after the call to
- * the UDF. This method is only called with fake elements.
- *
- * Note: Fake elements are always considered as triggered. Therefore this
- * method does not have a triggered parameter.
- *
- * @param datapoint
- * the current fake data point
- * @param bufferSize
- * the current size of the buffer (only real elements are
- * counted)
- * @return the number of elements to delete from the buffer (only real
- * elements are counted)
- */
- public int notifyEvictionWithFakeElement(Object datapoint, int bufferSize);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
deleted file mode 100644
index b3b6935..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-/**
- * This {@link ActiveEvictionPolicy} wraps around a non active
- * {@link EvictionPolicy}. It forwards all calls to
- * {@link ActiveEvictionPolicy#notifyEvictionWithFakeElement(Object, int)} to
- * {@link EvictionPolicy#notifyEviction(Object, boolean, int)} while the
- * triggered parameter will be set to true.
- *
- * @param <DATA>
- * The data type handled by this policy
- */
-public class ActiveEvictionPolicyWrapper<DATA> implements ActiveEvictionPolicy<DATA> {
-
- /**
- * Auto generated version ID
- */
- private static final long serialVersionUID = -7656558669799505882L;
- private EvictionPolicy<DATA> nestedPolicy;
-
- /**
- * Creates a wrapper which activates the eviction policy which is wrapped
- * in. This means that the nested policy will get called on fake elements as
- * well as on real elements.
- *
- * @param nestedPolicy
- * The policy which should be activated/wrapped in.
- */
- public ActiveEvictionPolicyWrapper(EvictionPolicy<DATA> nestedPolicy) {
- if (nestedPolicy == null) {
- throw new RuntimeException("The nested policy must not be null.");
- }
- this.nestedPolicy = nestedPolicy;
- }
-
- @Override
- public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
- return nestedPolicy.notifyEviction(datapoint, triggered, bufferSize);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public int notifyEvictionWithFakeElement(Object datapoint, int bufferSize) {
- return nestedPolicy.notifyEviction((DATA) datapoint, true, bufferSize);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
deleted file mode 100644
index c44be37..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-/**
- * In case an {@link ActiveTriggerPolicy} is used, it can implement own
- * {@link Runnable} classes. Such {@link Runnable} classes will be executed as
- * an own thread and can submit fake elements, to the element buffer at any
- * time.
- *
- * The factory method for runnables of the {@link ActiveTriggerPolicy} gets an
- * instance of this interface as parameter. The describes adding of elements can
- * be done by the runnable using the methods provided in this interface.
- *
- */
-public interface ActiveTriggerCallback {
-
- /**
- * Submits a new fake data point to the element buffer. Such a fake element
- * might be used to trigger at any time, but will never be included in the
- * result of the reduce function. The submission of a fake element causes
- * notifications only at the {@link ActiveTriggerPolicy} and
- * {@link ActiveEvictionPolicy} implementations.
- *
- * @param datapoint
- * the fake data point to be added
- */
- public void sendFakeElement(Object datapoint);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
deleted file mode 100644
index abe5298..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-
-/**
- * This interface extends the {@link TriggerPolicy} interface with functionality
- * for active triggers. Active triggers can act in two ways:
- *
- * 1) Whenever an element arrives at the invokable, the
- * {@link ActiveTriggerPolicy#preNotifyTrigger(Object)} method gets called
- * first. It can return zero ore more fake data points which will be added
- * before the currently arrived real element gets processed. This allows to
- * handle empty windows in time based windowing with an user defined
- * {@link Timestamp}. Triggers are not called on fake datapoint. A fake
- * datapoint is always considered as triggered.
- *
- * 2) An active trigger has a factory method for a runnable. This factory method
- * gets called at the start up of the invokable. The returned runnable will be
- * executed in its own thread and can submit fake elements at any time threw an
- * {@link ActiveTriggerCallback}. This allows to have time based triggers based
- * on any system internal time measure. Triggers are not called on fake
- * datapoint. A fake datapoints is always considered as triggered.
- *
- * @param <DATA>
- * The data type which can be handled by this policy
- */
-public interface ActiveTriggerPolicy<DATA> extends TriggerPolicy<DATA> {
-
- /**
- * Whenever an element arrives at the invokable, the
- * {@link ActiveTriggerPolicy#preNotifyTrigger(Object)} method gets called
- * first. It can return zero ore more fake data points which will be added
- * before the the currently arrived real element gets processed. This allows
- * to handle empty windows in time based windowing with an user defined
- * {@link Timestamp}. Triggers are not called on fake datapoints. A fake
- * datapoint is always considered as triggered.
- *
- * @param datapoint
- * the data point which arrived at the invokable
- * @return zero ore more fake data points which will be added before the the
- * currently arrived real element gets processed.
- */
- public Object[] preNotifyTrigger(DATA datapoint);
-
- /**
- * This is the factory method for a runnable. This factory method gets
- * called at the start up of the invokable. The returned runnable will be
- * executed in its own thread and can submit fake elements at any time threw
- * an {@link ActiveTriggerCallback}. This allows to have time based triggers
- * based on any system internal time measure. Triggers are not called on
- * fake datapoints. A fake datapoint is always considered as triggered.
- *
- * @param callback
- * A callback object which allows to add fake elements from
- * within the returned {@link Runnable}.
- * @return The runnable implementation or null in case there is no. In case
- * an {@link ActiveTriggerPolicy} is used, it can implement own
- * {@link Runnable} classes. Such {@link Runnable} classes will be
- * executed as an own thread and can submit fake elements, to the
- * element buffer at any time.
- */
- public Runnable createActiveTriggerRunnable(ActiveTriggerCallback callback);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
deleted file mode 100644
index 132b495..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
-
-/**
- * When used in {@link GroupedWindowInvokable}, eviction policies must
- * provide a clone method. Eviction policies get cloned to provide an own
- * instance for each group and respectively each individual element buffer as
- * groups maintain their own buffers with the elements belonging to the
- * respective group.
- *
- * This interface extends {@link EvictionPolicy} with such a clone method. It
- * also adds the Java {@link Cloneable} interface as flag.
- *
- * @param <DATA>
- * The data type handled by this policy
- */
-public interface CloneableEvictionPolicy<DATA> extends EvictionPolicy<DATA>, Cloneable {
-
- /**
- * This method should return an exact copy of the object it belongs to
- * including the current object state.
- *
- * @return a copy of this object
- */
- public CloneableEvictionPolicy<DATA> clone();
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
deleted file mode 100644
index f5772a1..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
-
-/**
- * When used in {@link GroupedWindowInvokable}, trigger policies can provide
- * a clone method. Cloneable triggers can can be used in a distributed manner,
- * which means they get cloned to provide an own instance for each group. This
- * allows each group to trigger individually and only based on the elements
- * belonging to the respective group.
- *
- * This interface extends {@link TriggerPolicy} with such a clone method. It
- * also adds the Java {@link Cloneable} interface as flag.
- *
- * @param <DATA>
- * The data type handled by this policy
- */
-public interface CloneableTriggerPolicy<DATA> extends TriggerPolicy<DATA>, Cloneable {
-
- /**
- * This method should return an exact copy of the object it belongs to
- * including the current object state.
- *
- * @return a copy of this object
- */
- public CloneableTriggerPolicy<DATA> clone();
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
deleted file mode 100644
index a9e7b3f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-/**
- * This eviction policy allows the eviction of data points from the buffer using
- * a counter of arriving elements and a threshold (maximal buffer size)
- *
- * By default this policy does not react on fake elements. Wrap it in an
- * {@link ActiveEvictionPolicyWrapper} to make it count even fake elements.
- *
- * @param <IN>
- * the type of the incoming data points
- */
-public class CountEvictionPolicy<IN> implements CloneableEvictionPolicy<IN> {
-
- /**
- * Auto generated version id
- */
- private static final long serialVersionUID = 2319201348806427996L;
-
- int maxElements;
- int counter;
- int deleteOnEviction = 1;
-
- /**
- * This constructor allows the setup of the simplest possible count based
- * eviction. It keeps the size of the buffer according to the given
- * maxElements parameter by deleting the oldest element in the buffer.
- * Eviction only takes place if the counter of arriving elements would be
- * higher than maxElements without eviction.
- *
- * @param maxElements
- * The maximum number of elements before eviction. As soon as one
- * more element arrives, the oldest element will be deleted
- */
- public CountEvictionPolicy(int maxElements) {
- this(maxElements, 1);
- }
-
- /**
- * This constructor allows to set up both, the maximum number of elements
- * and the number of elements to be deleted in case of an eviction.
- *
- * Eviction only takes place if the counter of arriving elements would be
- * higher than maxElements without eviction. In such a case deleteOnEviction
- * elements will be removed from the buffer.
- *
- * The counter of arriving elements is adjusted respectively, but never set
- * below zero:
- * counter=(counter-deleteOnEviction<0)?0:counter-deleteOnEviction
- *
- * @param maxElements
- * maxElements The maximum number of elements before eviction.
- * @param deleteOnEviction
- * The number of elements to be deleted on eviction. The counter
- * will be adjusted respectively but never below zero.
- */
- public CountEvictionPolicy(int maxElements, int deleteOnEviction) {
- this(maxElements, deleteOnEviction, 0);
- }
-
- /**
- * The same as {@link CountEvictionPolicy#CountEvictionPolicy(int, int)}.
- * Additionally a custom start value for the counter of arriving elements
- * can be set. By setting a negative start value the first eviction can be
- * delayed.
- *
- * @param maxElements
- * maxElements The maximum number of elements before eviction.
- * @param deleteOnEviction
- * The number of elements to be deleted on eviction. The counter
- * will be adjusted respectively but never below zero.
- * @param startValue
- * A custom start value for the counter of arriving elements.
- * @see CountEvictionPolicy#NextGenCountEvictionPolicy(int, int)
- */
- public CountEvictionPolicy(int maxElements, int deleteOnEviction, int startValue) {
- this.counter = startValue;
- this.deleteOnEviction = deleteOnEviction;
- this.maxElements = maxElements;
- }
-
- @Override
- public int notifyEviction(IN datapoint, boolean triggered, int bufferSize) {
- // The comparison have to be >= and not == to cover case max=0
- if (counter >= maxElements) {
- // Adjust the counter according to the current eviction
- counter = (counter - deleteOnEviction < 0) ? 0 : counter - deleteOnEviction;
- // The current element will be added after the eviction
- // Therefore, increase counter in any case
- counter++;
- return deleteOnEviction;
- } else {
- counter++;
- return 0;
- }
- }
-
- @Override
- public CountEvictionPolicy<IN> clone() {
- return new CountEvictionPolicy<IN>(maxElements, deleteOnEviction, counter);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
deleted file mode 100644
index 7a3c75a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-/**
- * This policy triggers at every n'th element.
- *
- * @param <IN>
- * The type of the data points which are handled by this policy
- */
-public class CountTriggerPolicy<IN> implements CloneableTriggerPolicy<IN> {
-
- /**
- * Auto generated version ID
- */
- private static final long serialVersionUID = -6357200688886103968L;
-
- public static final int DEFAULT_START_VALUE = 0;
-
- private int counter;
- private int max;
-
- /**
- * This constructor will set up a count based trigger, which triggers after
- * max elements have arrived.
- *
- * @param max
- * The number of arriving elements before the trigger occurs.
- */
- public CountTriggerPolicy(int max) {
- this(max, DEFAULT_START_VALUE);
- }
-
- /**
- * In addition to {@link CountTriggerPolicy#CountTriggerPolicy(int)} this
- * constructor allows to set a custom start value for the element counter.
- * This can be used to delay the first trigger by setting a negative start
- * value. Often the first trigger should be delayed in case of sliding
- * windows. For example if the size of a window should be 4 and a trigger
- * should happen every 2, a start value of -2 would allow to also have the
- * first window of size 4.
- *
- * @param max
- * The number of arriving elements before the trigger occurs.
- * @param startValue
- * The start value for the counter of arriving elements.
- * @see CountTriggerPolicy#CountTriggerPolicy(int)
- */
- public CountTriggerPolicy(int max, int startValue) {
- this.max = max;
- this.counter = startValue;
- }
-
- @Override
- public boolean notifyTrigger(IN datapoint) {
- // The comparison have to be >= and not == to cover case max=0
- if (counter >= max) {
- // The current data point will be part of the next window!
- // Therefore the counter needs to be set to one already.
- counter = 1;
- return true;
- } else {
- counter++;
- return false;
- }
- }
-
- @Override
- public CountTriggerPolicy<IN> clone() {
- return new CountTriggerPolicy<IN>(max, counter);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
deleted file mode 100644
index 77bc692..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.windowing.policy;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
-
-/**
- * This policy calculates a delta between the data point which triggered last
- * and the currently arrived data point. It triggers if the delta is higher than
- * a specified threshold.
- *
- * In case it gets used for eviction, this policy starts from the first element
- * of the buffer and removes all elements from the buffer which have a higher
- * delta then the threshold. As soon as there is an element with a lower delta,
- * the eviction stops.
- *
- * By default this policy does not react on fake elements. Wrap it in an
- * {@link ActiveEvictionPolicyWrapper} to make it calculate the delta even on
- * fake elements.
- *
- * @param <DATA>
- * The type of the data points which are handled by this policy
- */
-public class DeltaPolicy<DATA> implements CloneableTriggerPolicy<DATA>,
- CloneableEvictionPolicy<DATA> {
-
- /**
- * Auto generated version ID
- */
- private static final long serialVersionUID = -7797538922123394967L;
-
- private DeltaFunction<DATA> deltaFuntion;
- private List<DATA> windowBuffer;
- private double threshold;
- private DATA triggerDataPoint;
-
- /**
- * Crates a delta policy which calculates a delta between the data point
- * which triggered last and the currently arrived data point. It triggers if
- * the delta is higher than a specified threshold.
- *
- * In case it gets used for eviction, this policy starts from the first
- * element of the buffer and removes all elements from the buffer which have
- * a higher delta then the threshold. As soon as there is an element with a
- * lower delta, the eviction stops.
- *
- * @param deltaFuntion
- * The delta function to be used.
- * @param init
- * The initial to be used for the calculation of a delta before
- * the first trigger.
- * @param threshold
- * The threshold upon which a triggering should happen.
- */
- public DeltaPolicy(DeltaFunction<DATA> deltaFuntion, DATA init, double threshold) {
- this.deltaFuntion = deltaFuntion;
- this.triggerDataPoint = init;
- this.windowBuffer = new LinkedList<DATA>();
- this.threshold = threshold;
- }
-
- @Override
- public boolean notifyTrigger(DATA datapoint) {
- if (deltaFuntion.getDelta(this.triggerDataPoint, datapoint) > this.threshold) {
- this.triggerDataPoint = datapoint;
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
- windowBuffer = windowBuffer.subList(windowBuffer.size() - bufferSize, bufferSize);
- int evictCount = 0;
- for (DATA bufferPoint : windowBuffer) {
- if (deltaFuntion.getDelta(bufferPoint, datapoint) < this.threshold) {
- break;
- }
- evictCount++;
- }
-
- if (evictCount > 0) {
- windowBuffer = windowBuffer.subList(evictCount, windowBuffer.size());
- }
- windowBuffer.add(datapoint);
- return evictCount;
- }
-
- @Override
- public DeltaPolicy<DATA> clone() {
- return new DeltaPolicy<DATA>(deltaFuntion, triggerDataPoint, threshold);
- }
-}
[38/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java
deleted file mode 100644
index 1c273d3..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.aggregation;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-
-public abstract class AggregationFunction<T> implements ReduceFunction<T> {
- private static final long serialVersionUID = 1L;
-
- public int position;
-
- public AggregationFunction(int pos) {
- this.position = pos;
- }
-
- public static enum AggregationType {
- SUM, MIN, MAX, MINBY, MAXBY,
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
deleted file mode 100644
index 226c45a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.aggregation;
-
-import java.lang.reflect.Array;
-import java.lang.reflect.Field;
-import java.util.List;
-
-import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
-import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.CompositeType;
-import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.typeutils.PojoTypeInfo;
-import org.apache.flink.api.java.typeutils.runtime.PojoComparator;
-
-public abstract class ComparableAggregator<T> extends AggregationFunction<T> {
-
- private static final long serialVersionUID = 1L;
-
- public Comparator comparator;
- public boolean byAggregate;
- public boolean first;
-
- public ComparableAggregator(int pos, AggregationType aggregationType, boolean first) {
- super(pos);
- this.comparator = Comparator.getForAggregation(aggregationType);
- this.byAggregate = (aggregationType == AggregationType.MAXBY)
- || (aggregationType == AggregationType.MINBY);
- this.first = first;
- }
-
- public static <R> AggregationFunction<R> getAggregator(int positionToAggregate,
- TypeInformation<R> typeInfo, AggregationType aggregationType) {
- return getAggregator(positionToAggregate, typeInfo, aggregationType, false);
- }
-
- public static <R> AggregationFunction<R> getAggregator(int positionToAggregate,
- TypeInformation<R> typeInfo, AggregationType aggregationType, boolean first) {
-
- if (typeInfo.isTupleType()) {
- return new TupleComparableAggregator<R>(positionToAggregate, aggregationType, first);
- } else if (typeInfo instanceof BasicArrayTypeInfo
- || typeInfo instanceof PrimitiveArrayTypeInfo) {
- return new ArrayComparableAggregator<R>(positionToAggregate, aggregationType, first);
- } else {
- return new SimpleComparableAggregator<R>(aggregationType);
- }
- }
-
- public static <R> AggregationFunction<R> getAggregator(String field,
- TypeInformation<R> typeInfo, AggregationType aggregationType, boolean first) {
-
- return new PojoComparableAggregator<R>(field, typeInfo, aggregationType, first);
- }
-
- private static class TupleComparableAggregator<T> extends ComparableAggregator<T> {
-
- private static final long serialVersionUID = 1L;
-
- public TupleComparableAggregator(int pos, AggregationType aggregationType, boolean first) {
- super(pos, aggregationType, first);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public T reduce(T value1, T value2) throws Exception {
- Tuple tuple1 = (Tuple) value1;
- Tuple tuple2 = (Tuple) value2;
-
- Comparable<Object> o1 = tuple1.getField(position);
- Object o2 = tuple2.getField(position);
-
- int c = comparator.isExtremal(o1, o2);
-
- if (byAggregate) {
- if (c == 1) {
- return (T) tuple1;
- }
- if (first) {
- if (c == 0) {
- return (T) tuple1;
- }
- }
-
- return (T) tuple2;
-
- } else {
- if (c == 1) {
- tuple2.setField(o1, position);
- }
- return (T) tuple2;
- }
-
- }
- }
-
- private static class ArrayComparableAggregator<T> extends ComparableAggregator<T> {
-
- private static final long serialVersionUID = 1L;
-
- public ArrayComparableAggregator(int pos, AggregationType aggregationType, boolean first) {
- super(pos, aggregationType, first);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public T reduce(T array1, T array2) throws Exception {
-
- Object v1 = Array.get(array1, position);
- Object v2 = Array.get(array2, position);
-
- int c = comparator.isExtremal((Comparable<Object>) v1, v2);
-
- if (byAggregate) {
- if (c == 1) {
- return array1;
- }
- if (first) {
- if (c == 0) {
- return array1;
- }
- }
-
- return array2;
- } else {
- if (c == 1) {
- Array.set(array2, position, v1);
- }
-
- return array2;
- }
- }
-
- }
-
- private static class SimpleComparableAggregator<T> extends ComparableAggregator<T> {
-
- private static final long serialVersionUID = 1L;
-
- public SimpleComparableAggregator(AggregationType aggregationType) {
- super(0, aggregationType, false);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public T reduce(T value1, T value2) throws Exception {
-
- if (comparator.isExtremal((Comparable<Object>) value1, value2) == 1) {
- return value1;
- } else {
- return value2;
- }
- }
-
- }
-
- private static class PojoComparableAggregator<T> extends ComparableAggregator<T> {
-
- private static final long serialVersionUID = 1L;
- PojoComparator<T> pojoComparator;
-
- public PojoComparableAggregator(String field, TypeInformation<?> typeInfo,
- AggregationType aggregationType, boolean first) {
- super(0, aggregationType, first);
- if (!(typeInfo instanceof CompositeType<?>)) {
- throw new IllegalArgumentException(
- "Key expressions are only supported on POJO types and Tuples. "
- + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
- }
-
- @SuppressWarnings("unchecked")
- CompositeType<T> cType = (CompositeType<T>) typeInfo;
-
- List<FlatFieldDescriptor> fieldDescriptors = cType.getFlatFields(field);
- int logicalKeyPosition = fieldDescriptors.get(0).getPosition();
-
- if (cType instanceof PojoTypeInfo) {
- pojoComparator = (PojoComparator<T>) cType.createComparator(
- new int[] { logicalKeyPosition }, new boolean[] { false }, 0);
- } else {
- throw new IllegalArgumentException(
- "Key expressions are only supported on POJO types. "
- + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
- }
- }
-
- @Override
- public T reduce(T value1, T value2) throws Exception {
-
- Field[] keyFields = pojoComparator.getKeyFields();
- Object field1 = pojoComparator.accessField(keyFields[0], value1);
- Object field2 = pojoComparator.accessField(keyFields[0], value2);
-
- @SuppressWarnings("unchecked")
- int c = comparator.isExtremal((Comparable<Object>) field1, field2);
-
- if (byAggregate) {
- if (c == 1) {
- return value1;
- }
- if (first) {
- if (c == 0) {
- return value1;
- }
- }
-
- return value2;
- } else {
- if (c == 1) {
- keyFields[0].set(value2, field1);
- }
-
- return value2;
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/Comparator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/Comparator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/Comparator.java
deleted file mode 100644
index f56774b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/Comparator.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.aggregation;
-
-import java.io.Serializable;
-
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
-
-public abstract class Comparator implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- public abstract <R> int isExtremal(Comparable<R> o1, R o2);
-
- public static Comparator getForAggregation(AggregationType type) {
- switch (type) {
- case MAX:
- return new MaxComparator();
- case MIN:
- return new MinComparator();
- case MINBY:
- return new MinByComparator();
- case MAXBY:
- return new MaxByComparator();
- default:
- throw new IllegalArgumentException("Unsupported aggregation type.");
- }
- }
-
- private static class MaxComparator extends Comparator {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public <R> int isExtremal(Comparable<R> o1, R o2) {
- return o1.compareTo(o2) > 0 ? 1 : 0;
- }
-
- }
-
- private static class MaxByComparator extends Comparator {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public <R> int isExtremal(Comparable<R> o1, R o2) {
- int c = o1.compareTo(o2);
- if (c > 0) {
- return 1;
- }
- if (c == 0) {
- return 0;
- } else {
- return -1;
- }
- }
-
- }
-
- private static class MinByComparator extends Comparator {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public <R> int isExtremal(Comparable<R> o1, R o2) {
- int c = o1.compareTo(o2);
- if (c < 0) {
- return 1;
- }
- if (c == 0) {
- return 0;
- } else {
- return -1;
- }
- }
-
- }
-
- private static class MinComparator extends Comparator {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public <R> int isExtremal(Comparable<R> o1, R o2) {
- return o1.compareTo(o2) < 0 ? 1 : 0;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java
deleted file mode 100644
index 142028b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.aggregation;
-
-import java.lang.reflect.Array;
-import java.lang.reflect.Field;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
-import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.CompositeType;
-import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.typeutils.PojoTypeInfo;
-import org.apache.flink.api.java.typeutils.runtime.PojoComparator;
-
-public abstract class SumAggregator {
-
- public static <T> ReduceFunction<T> getSumFunction(int pos, Class<?> clazz,
- TypeInformation<T> typeInfo) {
-
- if (typeInfo.isTupleType()) {
- return new TupleSumAggregator<T>(pos, SumFunction.getForClass(clazz));
- } else if (typeInfo instanceof BasicArrayTypeInfo
- || typeInfo instanceof PrimitiveArrayTypeInfo) {
- return new ArraySumAggregator<T>(pos, SumFunction.getForClass(clazz));
- } else {
- return new SimpleSumAggregator<T>(SumFunction.getForClass(clazz));
- }
-
- }
-
- public static <T> ReduceFunction<T> getSumFunction(String field, TypeInformation<T> typeInfo) {
-
- return new PojoSumAggregator<T>(field, typeInfo);
- }
-
- private static class TupleSumAggregator<T> extends AggregationFunction<T> {
-
- private static final long serialVersionUID = 1L;
-
- SumFunction adder;
-
- public TupleSumAggregator(int pos, SumFunction adder) {
- super(pos);
- this.adder = adder;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public T reduce(T value1, T value2) throws Exception {
-
- Tuple tuple1 = (Tuple) value1;
- Tuple tuple2 = (Tuple) value2;
-
- tuple2.setField(adder.add(tuple1.getField(position), tuple2.getField(position)),
- position);
-
- return (T) tuple2;
- }
-
- }
-
- private static class ArraySumAggregator<T> extends AggregationFunction<T> {
-
- private static final long serialVersionUID = 1L;
-
- SumFunction adder;
-
- public ArraySumAggregator(int pos, SumFunction adder) {
- super(pos);
- this.adder = adder;
- }
-
- @Override
- public T reduce(T value1, T value2) throws Exception {
-
- Object v1 = Array.get(value1, position);
- Object v2 = Array.get(value2, position);
- Array.set(value2, position, adder.add(v1, v2));
- return value2;
- }
-
- }
-
- private static class SimpleSumAggregator<T> extends AggregationFunction<T> {
-
- private static final long serialVersionUID = 1L;
-
- SumFunction adder;
-
- public SimpleSumAggregator(SumFunction adder) {
- super(0);
- this.adder = adder;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public T reduce(T value1, T value2) throws Exception {
-
- return (T) adder.add(value1, value2);
- }
-
- }
-
- private static class PojoSumAggregator<T> extends AggregationFunction<T> {
-
- private static final long serialVersionUID = 1L;
- SumFunction adder;
- PojoComparator<T> comparator;
-
- public PojoSumAggregator(String field, TypeInformation<?> type) {
- super(0);
- if (!(type instanceof CompositeType<?>)) {
- throw new IllegalArgumentException(
- "Key expressions are only supported on POJO types and Tuples. "
- + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
- }
-
- @SuppressWarnings("unchecked")
- CompositeType<T> cType = (CompositeType<T>) type;
-
- List<FlatFieldDescriptor> fieldDescriptors = cType.getFlatFields(field);
-
- int logicalKeyPosition = fieldDescriptors.get(0).getPosition();
- Class<?> keyClass = fieldDescriptors.get(0).getType().getTypeClass();
-
- adder = SumFunction.getForClass(keyClass);
-
- if (cType instanceof PojoTypeInfo) {
- comparator = (PojoComparator<T>) cType.createComparator(
- new int[] { logicalKeyPosition }, new boolean[] { false }, 0);
- } else {
- throw new IllegalArgumentException(
- "Key expressions are only supported on POJO types. "
- + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
- }
- }
-
- @Override
- public T reduce(T value1, T value2) throws Exception {
-
- Field[] keyFields = comparator.getKeyFields();
- Object field1 = comparator.accessField(keyFields[0], value1);
- Object field2 = comparator.accessField(keyFields[0], value2);
-
- keyFields[0].set(value2, adder.add(field1, field2));
-
- return value2;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java
deleted file mode 100644
index 2aef19c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.aggregation;
-
-import java.io.Serializable;
-
-public abstract class SumFunction implements Serializable{
-
- private static final long serialVersionUID = 1L;
-
- public abstract Object add(Object o1, Object o2);
-
- public static SumFunction getForClass(Class<?> clazz) {
-
- if (clazz == Integer.class) {
- return new IntSum();
- } else if (clazz == Long.class) {
- return new LongSum();
- } else if (clazz == Short.class) {
- return new ShortSum();
- } else if (clazz == Double.class) {
- return new DoubleSum();
- } else if (clazz == Float.class) {
- return new FloatSum();
- } else if (clazz == Byte.class) {
- return new ByteSum();
- } else {
- throw new RuntimeException("DataStream cannot be summed because the class "
- + clazz.getSimpleName() + " does not support the + operator.");
- }
- }
-
- public static class IntSum extends SumFunction {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Object add(Object value1, Object value2) {
- return (Integer) value1 + (Integer) value2;
- }
- }
-
- public static class LongSum extends SumFunction {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Object add(Object value1, Object value2) {
- return (Long) value1 + (Long) value2;
- }
- }
-
- public static class DoubleSum extends SumFunction {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Object add(Object value1, Object value2) {
- return (Double) value1 + (Double) value2;
- }
- }
-
- public static class ShortSum extends SumFunction {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Object add(Object value1, Object value2) {
- return (Short) value1 + (Short) value2;
- }
- }
-
- public static class FloatSum extends SumFunction {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Object add(Object value1, Object value2) {
- return (Float) value1 + (Float) value2;
- }
- }
-
- public static class ByteSum extends SumFunction {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Object add(Object value1, Object value2) {
- return (Byte) value1 + (Byte) value2;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoFlatMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoFlatMapFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoFlatMapFunction.java
deleted file mode 100644
index 10e8bac..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoFlatMapFunction.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.co;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.util.Collector;
-
-/**
- * A CoFlatMapFunction represents a FlatMap transformation with two different
- * input types.
- *
- * @param <IN1>
- * Type of the first input.
- * @param <IN2>
- * Type of the second input.
- * @param <OUT>
- * Output type.
- */
-public interface CoFlatMapFunction<IN1, IN2, OUT> extends Function, Serializable {
-
- public void flatMap1(IN1 value, Collector<OUT> out) throws Exception;
-
- public void flatMap2(IN2 value, Collector<OUT> out) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoMapFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoMapFunction.java
deleted file mode 100644
index 2903828..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoMapFunction.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.co;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.Function;
-
-/**
- * A CoMapFunction represents a Map transformation with two different input
- * types.
- *
- * @param <IN1>
- * Type of the first input.
- * @param <IN2>
- * Type of the second input.
- * @param <OUT>
- * Output type.
- */
-public interface CoMapFunction<IN1, IN2, OUT> extends Function, Serializable {
-
- public OUT map1(IN1 value);
-
- public OUT map2(IN2 value);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoReduceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoReduceFunction.java
deleted file mode 100644
index 879a1b4..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoReduceFunction.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.co;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.Function;
-
-/**
- * The CoReduceFunction interface represents a Reduce transformation with two
- * different input streams. The reduce1 function combine groups of elements of
- * the first input with the same key to a single value, while reduce2 combine
- * groups of elements of the second input with the same key to a single value.
- * Each produced values are mapped to the same type by map1 and map2,
- * respectively, to form one output stream.
- *
- * The basic syntax for using a grouped ReduceFunction is as follows:
- *
- * <pre>
- * <blockquote>
- * ConnectedDataStream<X> input = ...;
- *
- * ConnectedDataStream<X> result = input.groupBy(keyPosition1, keyPosition2)
- * .reduce(new MyCoReduceFunction(), keyPosition1, keyPosition2).addSink(...);
- * </blockquote>
- * </pre>
- * <p>
- *
- * @param <IN1>
- * Type of the first input.
- * @param <IN2>
- * Type of the second input.
- * @param <OUT>
- * Output type.
- */
-public interface CoReduceFunction<IN1, IN2, OUT> extends Function, Serializable {
-
- /**
- * The core method of CoReduceFunction, combining two values of the first
- * input into one value of the same type. The reduce1 function is
- * consecutively applied to all values of a group until only a single value
- * remains.
- *
- * @param value1
- * The first value to combine.
- * @param value2
- * The second value to combine.
- * @return The combined value of both input values.
- *
- * @throws Exception
- * This method may throw exceptions. Throwing an exception will
- * cause the operation to fail and may trigger recovery.
- */
- public IN1 reduce1(IN1 value1, IN1 value2);
-
- /**
- * The core method of ReduceFunction, combining two values of the second
- * input into one value of the same type. The reduce2 function is
- * consecutively applied to all values of a group until only a single value
- * remains.
- *
- * @param value1
- * The first value to combine.
- * @param value2
- * The second value to combine.
- * @return The combined value of both input values.
- *
- * @throws Exception
- * This method may throw exceptions. Throwing an exception will
- * cause the operation to fail and may trigger recovery.
- */
- public IN2 reduce2(IN2 value1, IN2 value2);
-
- /**
- * Maps the reduced first input to the output type.
- *
- * @param value
- * Type of the first input.
- * @return the output type.
- */
- public OUT map1(IN1 value);
-
- /**
- * Maps the reduced second input to the output type.
- *
- * @param value
- * Type of the second input.
- * @return the output type.
- */
- public OUT map2(IN2 value);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoWindowFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoWindowFunction.java
deleted file mode 100644
index 2f2514f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CoWindowFunction.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.co;
-
-import java.io.Serializable;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.util.Collector;
-
-public interface CoWindowFunction<IN1, IN2, O> extends Function, Serializable {
-
- public void coWindow(List<IN1> first, List<IN2> second, Collector<O> out) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java
deleted file mode 100644
index 9cafcd1..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.co;
-
-import java.util.List;
-
-import org.apache.flink.api.common.functions.CrossFunction;
-import org.apache.flink.util.Collector;
-
-public class CrossWindowFunction<IN1, IN2, OUT> implements CoWindowFunction<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
-
- private CrossFunction<IN1, IN2, OUT> crossFunction;
-
- public CrossWindowFunction(CrossFunction<IN1, IN2, OUT> crossFunction) {
- this.crossFunction = crossFunction;
- }
-
- @Override
- public void coWindow(List<IN1> first, List<IN2> second, Collector<OUT> out) throws Exception {
- for (IN1 firstValue : first) {
- for (IN2 secondValue : second) {
- out.collect(crossFunction.cross(firstValue, secondValue));
- }
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
deleted file mode 100644
index 9b39f33..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.co;
-
-import java.util.List;
-
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.util.Collector;
-
-public class JoinWindowFunction<IN1, IN2, OUT> implements CoWindowFunction<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
-
- private KeySelector<IN1, ?> keySelector1;
- private KeySelector<IN2, ?> keySelector2;
- private JoinFunction<IN1, IN2, OUT> joinFunction;
-
- public JoinWindowFunction(KeySelector<IN1, ?> keySelector1, KeySelector<IN2, ?> keySelector2,
- JoinFunction<IN1, IN2, OUT> joinFunction) {
- this.keySelector1 = keySelector1;
- this.keySelector2 = keySelector2;
- this.joinFunction = joinFunction;
- }
-
- @Override
- public void coWindow(List<IN1> first, List<IN2> second, Collector<OUT> out) throws Exception {
- for (IN1 item1 : first) {
- Object key1 = keySelector1.getKey(item1);
-
- for (IN2 item2 : second) {
- Object key2 = keySelector2.getKey(item2);
-
- if (key1.equals(key2)) {
- out.collect(joinFunction.join(item1, item2));
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoFlatMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoFlatMapFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoFlatMapFunction.java
deleted file mode 100644
index 2458f1b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoFlatMapFunction.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.co;
-
-import org.apache.flink.api.common.functions.AbstractRichFunction;
-import org.apache.flink.api.common.functions.RichFunction;
-
-/**
- * A RichCoFlatMapFunction represents a FlatMap transformation with two different input
- * types. In addition to that the user can use the features provided by the
- * {@link RichFunction} interface.
- *
- * @param <IN1>
- * Type of the first input.
- * @param <IN2>
- * Type of the second input.
- * @param <OUT>
- * Output type.
- */
-public abstract class RichCoFlatMapFunction<IN1, IN2, OUT> extends AbstractRichFunction implements
- CoFlatMapFunction<IN1, IN2, OUT> {
-
- private static final long serialVersionUID = 1L;
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoMapFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoMapFunction.java
deleted file mode 100755
index 20d520c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoMapFunction.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.co;
-
-import org.apache.flink.api.common.functions.AbstractRichFunction;
-import org.apache.flink.api.common.functions.RichFunction;
-
-/**
- * A RichCoMapFunction represents a Map transformation with two different input
- * types. In addition to that the user can use the features provided by the
- * {@link RichFunction} interface.
- *
- * @param <IN1>
- * Type of the first input.
- * @param <IN2>
- * Type of the second input.
- * @param <OUT>
- * Output type.
- */
-public abstract class RichCoMapFunction<IN1, IN2, OUT> extends AbstractRichFunction implements
- CoMapFunction<IN1, IN2, OUT> {
-
- private static final long serialVersionUID = 1L;
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoReduceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoReduceFunction.java
deleted file mode 100644
index 655923f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoReduceFunction.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.co;
-
-import org.apache.flink.api.common.functions.AbstractRichFunction;
-import org.apache.flink.api.common.functions.RichFunction;
-
-/**
- * A RichCoReduceFunction represents a Reduce transformation with two different input
- * types. In addition to that the user can use the features provided by the
- * {@link RichFunction} interface.
- *
- * @param <IN1>
- * Type of the first input.
- * @param <IN2>
- * Type of the second input.
- * @param <OUT>
- * Output type.
- */
-public abstract class RichCoReduceFunction<IN1, IN2, OUT> extends AbstractRichFunction implements
- CoReduceFunction<IN1, IN2, OUT> {
-
- private static final long serialVersionUID = 1L;
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoWindowFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoWindowFunction.java
deleted file mode 100644
index 2709203..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/RichCoWindowFunction.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.co;
-
-import java.util.List;
-
-import org.apache.flink.api.common.functions.AbstractRichFunction;
-import org.apache.flink.util.Collector;
-
-public abstract class RichCoWindowFunction<IN1, IN2, O> extends AbstractRichFunction implements
- CoWindowFunction<IN1, IN2, O> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public abstract void coWindow(List<IN1> first, List<IN2> second, Collector<O> out)
- throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
deleted file mode 100644
index 24beba1..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.flink.api.common.io.CleanupWhenUnsuccessful;
-import org.apache.flink.api.common.io.OutputFormat;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Simple implementation of the SinkFunction writing tuples in the specified
- * OutputFormat format. Tuples are collected to a list and written to the file
- * periodically. The target path and the overwrite mode are pre-packaged in
- * format.
- *
- * @param <IN>
- * Input type
- */
-public abstract class FileSinkFunction<IN> extends RichSinkFunction<IN> {
- private static final long serialVersionUID = 1L;
- private static final Logger LOG = LoggerFactory.getLogger(FileSinkFunction.class);
- protected ArrayList<IN> tupleList = new ArrayList<IN>();
- protected volatile OutputFormat<IN> format;
- protected volatile boolean cleanupCalled = false;
- protected int indexInSubtaskGroup;
- protected int currentNumberOfSubtasks;
-
- public FileSinkFunction(OutputFormat<IN> format) {
- this.format = format;
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
- format.configure(context.getTaskStubParameters());
- indexInSubtaskGroup = context.getIndexOfThisSubtask();
- currentNumberOfSubtasks = context.getNumberOfParallelSubtasks();
- format.open(indexInSubtaskGroup, currentNumberOfSubtasks);
- }
-
- @Override
- public void invoke(IN record) throws Exception {
- tupleList.add(record);
- if (updateCondition()) {
- flush();
- }
- }
-
- @Override
- public void close() throws IOException {
- if (!tupleList.isEmpty()) {
- flush();
- }
- try {
- format.close();
- } catch (Exception ex) {
- try {
- if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
- cleanupCalled = true;
- ((CleanupWhenUnsuccessful) format).tryCleanupOnError();
- }
- } catch (Throwable t) {
- LOG.error("Cleanup on error failed.", t);
- }
- }
- }
-
- protected void flush() {
- try {
- for (IN rec : tupleList) {
- format.writeRecord(rec);
- }
- } catch (Exception ex) {
- try {
- if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
- cleanupCalled = true;
- ((CleanupWhenUnsuccessful) format).tryCleanupOnError();
- }
- } catch (Throwable t) {
- LOG.error("Cleanup on error failed.", t);
- }
- }
- resetParameters();
- }
-
- /**
- * Condition for writing the contents of tupleList and clearing it.
- *
- * @return value of the updating condition
- */
- protected abstract boolean updateCondition();
-
- /**
- * Statements to be executed after writing a batch goes here.
- */
- protected abstract void resetParameters();
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
deleted file mode 100644
index f049a32..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-import org.apache.flink.api.common.io.OutputFormat;
-
-/**
- * Implementation of FileSinkFunction. Writes tuples to file in every millis
- * milliseconds.
- *
- * @param <IN>
- * Input type
- */
-public class FileSinkFunctionByMillis<IN> extends FileSinkFunction<IN> {
- private static final long serialVersionUID = 1L;
-
- private final long millis;
- private long lastTime;
-
- public FileSinkFunctionByMillis(OutputFormat<IN> format, long millis) {
- super(format);
- this.millis = millis;
- lastTime = System.currentTimeMillis();
- }
-
- /**
- * Condition for writing the contents of tupleList and clearing it.
- *
- * @return value of the updating condition
- */
- @Override
- protected boolean updateCondition() {
- return System.currentTimeMillis() - lastTime >= millis;
- }
-
- /**
- * Statements to be executed after writing a batch goes here.
- */
- @Override
- protected void resetParameters() {
- tupleList.clear();
- lastTime = System.currentTimeMillis();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/PrintSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/PrintSinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/PrintSinkFunction.java
deleted file mode 100755
index d460749..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/PrintSinkFunction.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-import java.io.PrintStream;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
-
-/**
- * Implementation of the SinkFunction writing every tuple to the standard
- * output or standard error stream.
- *
- * @param <IN>
- * Input record type
- */
-public class PrintSinkFunction<IN> extends RichSinkFunction<IN> {
- private static final long serialVersionUID = 1L;
-
- private static final boolean STD_OUT = false;
- private static final boolean STD_ERR = true;
-
- private boolean target;
- private transient PrintStream stream;
- private transient String prefix;
-
- /**
- * Instantiates a print sink function that prints to standard out.
- */
- public PrintSinkFunction() {}
-
- /**
- * Instantiates a print sink function that prints to standard out.
- *
- * @param stdErr True, if the format should print to standard error instead of standard out.
- */
- public PrintSinkFunction(boolean stdErr) {
- target = stdErr;
- }
-
- public void setTargetToStandardOut() {
- target = STD_OUT;
- }
-
- public void setTargetToStandardErr() {
- target = STD_ERR;
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
- // get the target stream
- stream = target == STD_OUT ? System.out : System.err;
-
- // set the prefix if we have a >1 DOP
- prefix = (context.getNumberOfParallelSubtasks() > 1) ?
- ((context.getIndexOfThisSubtask() + 1) + "> ") : null;
- }
-
- @Override
- public void invoke(IN record) {
- if (prefix != null) {
- stream.println(prefix + record.toString());
- }
- else {
- stream.println(record.toString());
- }
- }
-
- @Override
- public void close() throws Exception {
- this.stream = null;
- this.prefix = null;
- super.close();
- }
-
- @Override
- public String toString() {
- return "Print to " + (target == STD_OUT ? "System.out" : "System.err");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
deleted file mode 100755
index 3b8a4db..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-import org.apache.flink.api.common.functions.AbstractRichFunction;
-
-public abstract class RichSinkFunction<IN> extends AbstractRichFunction implements SinkFunction<IN> {
-
- private static final long serialVersionUID = 1L;
-
- public abstract void invoke(IN value) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
deleted file mode 100644
index 6097603..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.Function;
-
-public interface SinkFunction<IN> extends Function, Serializable {
-
- public abstract void invoke(IN value) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormat.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormat.java
deleted file mode 100644
index a606302..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormat.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-
-/**
- * Abstract class for formatting the output of the writeAsText and writeAsCsv
- * functions.
- *
- * @param <IN>
- * Input tuple type
- */
-public abstract class WriteFormat<IN> implements Serializable {
- private static final long serialVersionUID = 1L;
-
- /**
- * Writes the contents of tupleList to the file specified by path.
- *
- * @param path
- * is the path to the location where the tuples are written
- * @param tupleList
- * is the list of tuples to be written
- */
- protected abstract void write(String path, ArrayList<IN> tupleList);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java
deleted file mode 100644
index b22fd80..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-
-/**
- * Writes tuples in csv format.
- *
- * @param <IN>
- * Input tuple type
- */
-public class WriteFormatAsCsv<IN> extends WriteFormat<IN> {
- private static final long serialVersionUID = 1L;
-
- @Override
- protected void write(String path, ArrayList<IN> tupleList) {
- try {
- PrintWriter outStream = new PrintWriter(new BufferedWriter(new FileWriter(path, true)));
- for (IN tupleToWrite : tupleList) {
- String strTuple = tupleToWrite.toString();
- outStream.println(strTuple.substring(1, strTuple.length() - 1));
- }
- outStream.close();
- } catch (IOException e) {
- throw new RuntimeException("Exception occured while writing file " + path, e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsText.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsText.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsText.java
deleted file mode 100644
index 5891104..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsText.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-
-/**
- * Writes tuples in text format.
- *
- * @param <IN>
- * Input tuple type
- */
-public class WriteFormatAsText<IN> extends WriteFormat<IN> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void write(String path, ArrayList<IN> tupleList) {
- try {
- PrintWriter outStream = new PrintWriter(new BufferedWriter(new FileWriter(path, true)));
- for (IN tupleToWrite : tupleList) {
- outStream.println(tupleToWrite);
- }
- outStream.close();
- } catch (IOException e) {
- throw new RuntimeException("Exception occured while writing file " + path, e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java
deleted file mode 100644
index 0c52afc..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-import java.io.FileNotFoundException;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-
-/**
- * Simple implementation of the SinkFunction writing tuples as simple text to
- * the file specified by path. Tuples are collected to a list and written to the
- * file periodically. The file specified by path is created if it does not
- * exist, cleared if it exists before the writing.
- *
- * @param <IN>
- * Input tuple type
- */
-public abstract class WriteSinkFunction<IN> implements SinkFunction<IN> {
- private static final long serialVersionUID = 1L;
-
- protected final String path;
- protected ArrayList<IN> tupleList = new ArrayList<IN>();
- protected WriteFormat<IN> format;
-
- public WriteSinkFunction(String path, WriteFormat<IN> format) {
- this.path = path;
- this.format = format;
- cleanFile(path);
- }
-
- /**
- * Creates target file if it does not exist, cleans it if it exists.
- *
- * @param path
- * is the path to the location where the tuples are written
- */
- protected void cleanFile(String path) {
- try {
- PrintWriter writer;
- writer = new PrintWriter(path);
- writer.print("");
- writer.close();
- } catch (FileNotFoundException e) {
- throw new RuntimeException("File not found " + path, e);
- }
- }
-
- /**
- * Condition for writing the contents of tupleList and clearing it.
- *
- * @return value of the updating condition
- */
- protected abstract boolean updateCondition();
-
- /**
- * Statements to be executed after writing a batch goes here.
- */
- protected abstract void resetParameters();
-
- /**
- * Implementation of the invoke method of the SinkFunction class. Collects
- * the incoming tuples in tupleList and appends the list to the end of the
- * target file if updateCondition() is true or the current tuple is the
- * endTuple.
- */
- @Override
- public void invoke(IN tuple) {
-
- tupleList.add(tuple);
- if (updateCondition()) {
- format.write(path, tupleList);
- resetParameters();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java
deleted file mode 100644
index ee6df94..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-/**
- * Implementation of WriteSinkFunction. Writes tuples to file in every millis
- * milliseconds.
- *
- * @param <IN>
- * Input tuple type
- */
-public class WriteSinkFunctionByMillis<IN> extends WriteSinkFunction<IN> {
- private static final long serialVersionUID = 1L;
-
- private final long millis;
- private long lastTime;
-
- public WriteSinkFunctionByMillis(String path, WriteFormat<IN> format, long millis) {
- super(path, format);
- this.millis = millis;
- lastTime = System.currentTimeMillis();
- }
-
- @Override
- protected boolean updateCondition() {
- return System.currentTimeMillis() - lastTime >= millis;
- }
-
- @Override
- protected void resetParameters() {
- tupleList.clear();
- lastTime = System.currentTimeMillis();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
deleted file mode 100644
index 05a2489..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileMonitoringFunction.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.core.fs.FileStatus;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FileMonitoringFunction implements SourceFunction<Tuple3<String, Long, Long>> {
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(FileMonitoringFunction.class);
-
- public enum WatchType {
- ONLY_NEW_FILES, // Only new files will be processed.
- REPROCESS_WITH_APPENDED, // When some files are appended, all contents of the files will be processed.
- PROCESS_ONLY_APPENDED // When some files are appended, only appended contents will be processed.
- }
-
- private String path;
- private long interval;
- private WatchType watchType;
-
- private FileSystem fileSystem;
- private Map<String, Long> offsetOfFiles;
- private Map<String, Long> modificationTimes;
-
- public FileMonitoringFunction(String path, long interval, WatchType watchType) {
- this.path = path;
- this.interval = interval;
- this.watchType = watchType;
- this.modificationTimes = new HashMap<String, Long>();
- this.offsetOfFiles = new HashMap<String, Long>();
- }
-
- @Override
- public void invoke(Collector<Tuple3<String, Long, Long>> collector) throws Exception {
- fileSystem = FileSystem.get(new URI(path));
-
- while (true) {
- List<String> files = listNewFiles();
- for (String filePath : files) {
- if (watchType == WatchType.ONLY_NEW_FILES
- || watchType == WatchType.REPROCESS_WITH_APPENDED) {
- collector.collect(new Tuple3<String, Long, Long>(filePath, 0L, -1L));
- offsetOfFiles.put(filePath, -1L);
- } else if (watchType == WatchType.PROCESS_ONLY_APPENDED) {
- long offset = 0;
- long fileSize = fileSystem.getFileStatus(new Path(filePath)).getLen();
- if (offsetOfFiles.containsKey(filePath)) {
- offset = offsetOfFiles.get(filePath);
- }
-
- collector.collect(new Tuple3<String, Long, Long>(filePath, offset, fileSize));
- offsetOfFiles.put(filePath, fileSize);
-
- LOG.info("File processed: {}, {}, {}", filePath, offset, fileSize);
- }
- }
-
- Thread.sleep(interval);
- }
- }
-
- private List<String> listNewFiles() throws IOException {
- List<String> files = new ArrayList<String>();
-
- FileStatus[] statuses = fileSystem.listStatus(new Path(path));
-
- for (FileStatus status : statuses) {
- Path filePath = status.getPath();
- String fileName = filePath.getName();
- long modificationTime = status.getModificationTime();
-
- if (!isFiltered(fileName, modificationTime)) {
- files.add(filePath.toString());
- modificationTimes.put(fileName, modificationTime);
- }
- }
- return files;
- }
-
- private boolean isFiltered(String fileName, long modificationTime) {
-
- if ((watchType == WatchType.ONLY_NEW_FILES && modificationTimes.containsKey(fileName))
- || fileName.startsWith(".") || fileName.contains("_COPYING_")) {
- return true;
- } else {
- Long lastModification = modificationTimes.get(fileName);
- if (lastModification == null) {
- return false;
- } else {
- return lastModification >= modificationTime;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java
deleted file mode 100644
index 0882d9e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileReadFunction.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.core.fs.FSDataInputStream;
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.util.Collector;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.net.URI;
-
-public class FileReadFunction implements FlatMapFunction<Tuple3<String, Long, Long>, String> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception {
- FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0));
- stream.seek(value.f1);
-
- BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
- String line;
-
- try {
- while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) {
- out.collect(line);
- }
- } finally {
- reader.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java
deleted file mode 100644
index 5dfe4b2..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.api.java.typeutils.runtime.RuntimeStatefulSerializerFactory;
-import org.apache.flink.api.java.typeutils.runtime.RuntimeStatelessSerializerFactory;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.io.InputSplit;
-import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
-import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
-import org.apache.flink.util.Collector;
-
-public class FileSourceFunction extends RichSourceFunction<String> {
- private static final long serialVersionUID = 1L;
-
- private InputSplitProvider provider;
-
- private InputFormat<String, ?> inputFormat;
-
- private TypeSerializerFactory<String> serializerFactory;
-
- public FileSourceFunction(InputFormat<String, ?> format, TypeInformation<String> typeInfo) {
- this.inputFormat = format;
- this.serializerFactory = createSerializer(typeInfo);
- }
-
- private static TypeSerializerFactory<String> createSerializer(TypeInformation<String> typeInfo) {
- TypeSerializer<String> serializer = typeInfo.createSerializer();
-
- if (serializer.isStateful()) {
- return new RuntimeStatefulSerializerFactory<String>(serializer, typeInfo.getTypeClass());
- } else {
- return new RuntimeStatelessSerializerFactory<String>(serializer,
- typeInfo.getTypeClass());
- }
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
- this.provider = context.getInputSplitProvider();
- inputFormat.configure(context.getTaskStubParameters());
- }
-
- @Override
- public void invoke(Collector<String> collector) throws Exception {
- final TypeSerializer<String> serializer = serializerFactory.getSerializer();
- final Iterator<InputSplit> splitIterator = getInputSplits();
- @SuppressWarnings("unchecked")
- final InputFormat<String, InputSplit> format = (InputFormat<String, InputSplit>) this.inputFormat;
- try {
- while (splitIterator.hasNext()) {
-
- final InputSplit split = splitIterator.next();
- String record = serializer.createInstance();
-
- format.open(split);
- try {
- while (!format.reachedEnd()) {
- if ((record = format.nextRecord(record)) != null) {
- collector.collect(record);
- }
- }
- } finally {
- format.close();
- }
- }
- collector.close();
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
-
- private Iterator<InputSplit> getInputSplits() {
-
- return new Iterator<InputSplit>() {
-
- private InputSplit nextSplit;
-
- private boolean exhausted;
-
- @Override
- public boolean hasNext() {
- if (exhausted) {
- return false;
- }
-
- if (nextSplit != null) {
- return true;
- }
-
- InputSplit split = provider.getNextInputSplit();
-
- if (split != null) {
- this.nextSplit = split;
- return true;
- } else {
- exhausted = true;
- return false;
- }
- }
-
- @Override
- public InputSplit next() {
- if (this.nextSplit == null && !hasNext()) {
- throw new NoSuchElementException();
- }
-
- final InputSplit tmp = this.nextSplit;
- this.nextSplit = null;
- return tmp;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-}
[42/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
deleted file mode 100644
index 1d51216..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.SerializationException;
-import org.apache.commons.lang3.SerializationUtils;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.api.streamvertex.StreamVertexException;
-import org.apache.flink.streaming.partitioner.ShufflePartitioner;
-import org.apache.flink.streaming.partitioner.StreamPartitioner;
-import org.apache.flink.streaming.state.OperatorState;
-import org.apache.flink.util.InstantiationUtil;
-
-public class StreamConfig implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private static final String INPUT_TYPE = "inputType_";
- private static final String NUMBER_OF_OUTPUTS = "numberOfOutputs";
- private static final String NUMBER_OF_INPUTS = "numberOfInputs";
- private static final String CHAINED_OUTPUTS = "chainedOutputs";
- private static final String CHAINED_TASK_CONFIG = "chainedTaskConfig_";
- private static final String IS_CHAINED_VERTEX = "isChainedSubtask";
- private static final String OUTPUT_NAME = "outputName_";
- private static final String PARTITIONER_OBJECT = "partitionerObject_";
- private static final String VERTEX_NAME = "vertexName";
- private static final String ITERATION_ID = "iteration-id";
- private static final String OUTPUT_SELECTOR = "outputSelector";
- private static final String DIRECTED_EMIT = "directedEmit";
- private static final String SERIALIZEDUDF = "serializedudf";
- private static final String USER_FUNCTION = "userfunction";
- private static final String BUFFER_TIMEOUT = "bufferTimeout";
- private static final String OPERATOR_STATES = "operatorStates";
- private static final String TYPE_SERIALIZER_IN_1 = "typeSerializer_in_1";
- private static final String TYPE_SERIALIZER_IN_2 = "typeSerializer_in_2";
- private static final String TYPE_SERIALIZER_OUT_1 = "typeSerializer_out_1";
- private static final String TYPE_SERIALIZER_OUT_2 = "typeSerializer_out_2";
- private static final String ITERATON_WAIT = "iterationWait";
- private static final String OUTPUTS = "outVertexNames";
- private static final String EDGES_IN_ORDER = "rwOrder";
-
- // DEFAULT VALUES
-
- private static final long DEFAULT_TIMEOUT = 100;
-
- // CONFIG METHODS
-
- private Configuration config;
-
- public StreamConfig(Configuration config) {
- this.config = config;
- }
-
- public Configuration getConfiguration() {
- return config;
- }
-
- public void setVertexName(String vertexName) {
- config.setString(VERTEX_NAME, vertexName);
- }
-
- public String getTaskName() {
- return config.getString(VERTEX_NAME, "Missing");
- }
-
- public void setTypeSerializerIn1(StreamRecordSerializer<?> serializer) {
- setTypeSerializer(TYPE_SERIALIZER_IN_1, serializer);
- }
-
- public void setTypeSerializerIn2(StreamRecordSerializer<?> serializer) {
- setTypeSerializer(TYPE_SERIALIZER_IN_2, serializer);
- }
-
- public void setTypeSerializerOut1(StreamRecordSerializer<?> serializer) {
- setTypeSerializer(TYPE_SERIALIZER_OUT_1, serializer);
- }
-
- public void setTypeSerializerOut2(StreamRecordSerializer<?> serializer) {
- setTypeSerializer(TYPE_SERIALIZER_OUT_2, serializer);
- }
-
- @SuppressWarnings("unchecked")
- public <T> StreamRecordSerializer<T> getTypeSerializerIn1(ClassLoader cl) {
- try {
- return (StreamRecordSerializer<T>) InstantiationUtil.readObjectFromConfig(this.config,
- TYPE_SERIALIZER_IN_1, cl);
- } catch (Exception e) {
- throw new RuntimeException("Could not instantiate serializer.");
- }
- }
-
- @SuppressWarnings("unchecked")
- public <T> StreamRecordSerializer<T> getTypeSerializerIn2(ClassLoader cl) {
- try {
- return (StreamRecordSerializer<T>) InstantiationUtil.readObjectFromConfig(this.config,
- TYPE_SERIALIZER_IN_2, cl);
- } catch (Exception e) {
- throw new RuntimeException("Could not instantiate serializer.");
- }
- }
-
- @SuppressWarnings("unchecked")
- public <T> StreamRecordSerializer<T> getTypeSerializerOut1(ClassLoader cl) {
- try {
- return (StreamRecordSerializer<T>) InstantiationUtil.readObjectFromConfig(this.config,
- TYPE_SERIALIZER_OUT_1, cl);
- } catch (Exception e) {
- throw new RuntimeException("Could not instantiate serializer.");
- }
- }
-
- @SuppressWarnings("unchecked")
- public <T> StreamRecordSerializer<T> getTypeSerializerOut2(ClassLoader cl) {
- try {
- return (StreamRecordSerializer<T>) InstantiationUtil.readObjectFromConfig(this.config,
- TYPE_SERIALIZER_OUT_2, cl);
- } catch (Exception e) {
- throw new RuntimeException("Could not instantiate serializer.");
- }
- }
-
- private void setTypeSerializer(String key, StreamRecordSerializer<?> typeWrapper) {
- config.setBytes(key, SerializationUtils.serialize(typeWrapper));
- }
-
- public void setBufferTimeout(long timeout) {
- config.setLong(BUFFER_TIMEOUT, timeout);
- }
-
- public long getBufferTimeout() {
- return config.getLong(BUFFER_TIMEOUT, DEFAULT_TIMEOUT);
- }
-
- public void setUserInvokable(StreamInvokable<?, ?> invokableObject) {
- if (invokableObject != null) {
- config.setClass(USER_FUNCTION, invokableObject.getClass());
-
- try {
- config.setBytes(SERIALIZEDUDF, SerializationUtils.serialize(invokableObject));
- } catch (SerializationException e) {
- throw new RuntimeException("Cannot serialize invokable object "
- + invokableObject.getClass(), e);
- }
- }
- }
-
- @SuppressWarnings({ "unchecked" })
- public <T> T getUserInvokable(ClassLoader cl) {
- try {
- return (T) InstantiationUtil.readObjectFromConfig(this.config, SERIALIZEDUDF, cl);
- } catch (Exception e) {
- throw new StreamVertexException("Cannot instantiate user function", e);
- }
- }
-
- public void setDirectedEmit(boolean directedEmit) {
- config.setBoolean(DIRECTED_EMIT, directedEmit);
- }
-
- public boolean isDirectedEmit() {
- return config.getBoolean(DIRECTED_EMIT, false);
- }
-
-
- public void setOutputSelectors(List<OutputSelector<?>> outputSelector) {
- try {
- if (outputSelector != null) {
- setDirectedEmit(true);
- config.setBytes(OUTPUT_SELECTOR, SerializationUtils.serialize((Serializable) outputSelector));
- }
- } catch (SerializationException e) {
- throw new RuntimeException("Cannot serialize OutputSelector");
- }
- }
-
- @SuppressWarnings("unchecked")
- public <T> List<OutputSelector<T>> getOutputSelectors(ClassLoader cl) {
- try {
- return (List<OutputSelector<T>>) InstantiationUtil.readObjectFromConfig(this.config,
- OUTPUT_SELECTOR, cl);
- } catch (Exception e) {
- throw new StreamVertexException("Cannot deserialize and instantiate OutputSelector", e);
- }
- }
-
- public void setIterationId(Integer iterationId) {
- config.setInteger(ITERATION_ID, iterationId);
- }
-
- public Integer getIterationId() {
- return config.getInteger(ITERATION_ID, 0);
- }
-
- public void setIterationWaitTime(long time) {
- config.setLong(ITERATON_WAIT, time);
- }
-
- public long getIterationWaitTime() {
- return config.getLong(ITERATON_WAIT, 0);
- }
-
- public <T> void setPartitioner(String output, StreamPartitioner<T> partitionerObject) {
-
- config.setBytes(PARTITIONER_OBJECT + output,
- SerializationUtils.serialize(partitionerObject));
- }
-
- @SuppressWarnings("unchecked")
- public <T> StreamPartitioner<T> getPartitioner(ClassLoader cl, String output) {
- StreamPartitioner<T> partitioner = null;
- try {
- partitioner = (StreamPartitioner<T>) InstantiationUtil.readObjectFromConfig(
- this.config, PARTITIONER_OBJECT + output, cl);
- } catch (Exception e) {
- throw new RuntimeException("Partitioner could not be instantiated.");
- }
- if (partitioner != null) {
- return partitioner;
- } else {
- return new ShufflePartitioner<T>();
- }
- }
-
- public void setSelectedNames(String output, List<String> selected) {
- if (selected != null) {
- config.setBytes(OUTPUT_NAME + output,
- SerializationUtils.serialize((Serializable) selected));
- } else {
- config.setBytes(OUTPUT_NAME + output,
- SerializationUtils.serialize((Serializable) new ArrayList<String>()));
- }
- }
-
- @SuppressWarnings("unchecked")
- public List<String> getSelectedNames(String output) {
- return (List<String>) SerializationUtils.deserialize(config.getBytes(OUTPUT_NAME + output,
- null));
- }
-
- public void setNumberOfInputs(int numberOfInputs) {
- config.setInteger(NUMBER_OF_INPUTS, numberOfInputs);
- }
-
- public int getNumberOfInputs() {
- return config.getInteger(NUMBER_OF_INPUTS, 0);
- }
-
- public void setNumberOfOutputs(int numberOfOutputs) {
- config.setInteger(NUMBER_OF_OUTPUTS, numberOfOutputs);
- }
-
- public int getNumberOfOutputs() {
- return config.getInteger(NUMBER_OF_OUTPUTS, 0);
- }
-
- public void setOutputs(List<String> outputVertexNames) {
- config.setBytes(OUTPUTS, SerializationUtils.serialize((Serializable) outputVertexNames));
- }
-
- @SuppressWarnings("unchecked")
- public List<String> getOutputs(ClassLoader cl) {
- try {
- return (List<String>) InstantiationUtil.readObjectFromConfig(this.config, OUTPUTS, cl);
- } catch (Exception e) {
- throw new RuntimeException("Could not instantiate outputs.");
- }
- }
-
- public void setOutEdgesInOrder(List<Tuple2<String, String>> outEdgeList) {
-
- config.setBytes(EDGES_IN_ORDER, SerializationUtils.serialize((Serializable) outEdgeList));
- }
-
- @SuppressWarnings("unchecked")
- public List<Tuple2<String, String>> getOutEdgesInOrder(ClassLoader cl) {
- try {
- return (List<Tuple2<String, String>>) InstantiationUtil.readObjectFromConfig(
- this.config, EDGES_IN_ORDER, cl);
- } catch (Exception e) {
- throw new RuntimeException("Could not instantiate outputs.");
- }
- }
-
- public void setInputIndex(int inputNumber, Integer inputTypeNumber) {
- config.setInteger(INPUT_TYPE + inputNumber++, inputTypeNumber);
- }
-
- public int getInputIndex(int inputNumber) {
- return config.getInteger(INPUT_TYPE + inputNumber, 0);
- }
-
- public void setOperatorStates(Map<String, OperatorState<?>> states) {
- config.setBytes(OPERATOR_STATES, SerializationUtils.serialize((Serializable) states));
- }
-
- @SuppressWarnings("unchecked")
- public Map<String, OperatorState<?>> getOperatorStates(ClassLoader cl) {
- try {
- return (Map<String, OperatorState<?>>) InstantiationUtil.readObjectFromConfig(
- this.config, OPERATOR_STATES, cl);
- } catch (Exception e) {
- throw new RuntimeException("Could not load operator state");
- }
- }
-
- public void setChainedOutputs(List<String> chainedOutputs) {
- config.setBytes(CHAINED_OUTPUTS,
- SerializationUtils.serialize((Serializable) chainedOutputs));
- }
-
- @SuppressWarnings("unchecked")
- public List<String> getChainedOutputs(ClassLoader cl) {
- try {
- return (List<String>) InstantiationUtil.readObjectFromConfig(this.config,
- CHAINED_OUTPUTS, cl);
- } catch (Exception e) {
- throw new RuntimeException("Could not instantiate chained outputs.");
- }
- }
-
- public void setTransitiveChainedTaskConfigs(Map<String, StreamConfig> chainedTaskConfigs) {
- config.setBytes(CHAINED_TASK_CONFIG,
- SerializationUtils.serialize((Serializable) chainedTaskConfigs));
- }
-
- @SuppressWarnings("unchecked")
- public Map<String, StreamConfig> getTransitiveChainedTaskConfigs(ClassLoader cl) {
- try {
-
- Map<String, StreamConfig> confs = (Map<String, StreamConfig>) InstantiationUtil
- .readObjectFromConfig(this.config, CHAINED_TASK_CONFIG, cl);
-
- return confs == null ? new HashMap<String, StreamConfig>() : confs;
- } catch (Exception e) {
- throw new RuntimeException("Could not instantiate configuration.");
- }
- }
-
- public void setChainStart() {
- config.setBoolean(IS_CHAINED_VERTEX, true);
- }
-
- public boolean isChainStart() {
- return config.getBoolean(IS_CHAINED_VERTEX, false);
- }
-
- @Override
- public String toString() {
-
- ClassLoader cl = getClass().getClassLoader();
-
- StringBuilder builder = new StringBuilder();
- builder.append("\n=======================");
- builder.append("Stream Config");
- builder.append("=======================");
- builder.append("\nTask name: " + getTaskName());
- builder.append("\nNumber of non-chained inputs: " + getNumberOfInputs());
- builder.append("\nNumber of non-chained outputs: " + getNumberOfOutputs());
- builder.append("\nOutput names: " + getOutputs(cl));
- builder.append("\nPartitioning:");
- for (String outputname : getOutputs(cl)) {
- builder.append("\n\t" + outputname + ": "
- + getPartitioner(cl, outputname).getClass().getSimpleName());
- }
-
- builder.append("\nChained subtasks: " + getChainedOutputs(cl));
-
- try {
- builder.append("\nInvokable: " + getUserInvokable(cl).getClass().getSimpleName());
- } catch (Exception e) {
- builder.append("\nInvokable: Missing");
- }
- builder.append("\nBuffer timeout: " + getBufferTimeout());
- if (isChainStart() && getChainedOutputs(cl).size() > 0) {
- builder.append("\n\n\n---------------------\nChained task configs\n---------------------\n");
- builder.append(getTransitiveChainedTaskConfigs(cl)).toString();
- }
-
- return builder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
deleted file mode 100644
index b5e43af..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
+++ /dev/null
@@ -1,620 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.compiler.plan.StreamingPlan;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.api.streamvertex.CoStreamVertex;
-import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
-import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
-import org.apache.flink.streaming.api.streamvertex.StreamVertex;
-import org.apache.flink.streaming.partitioner.StreamPartitioner;
-import org.apache.flink.streaming.state.OperatorState;
-import org.apache.sling.commons.json.JSONArray;
-import org.apache.sling.commons.json.JSONException;
-import org.apache.sling.commons.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Object for building Apache Flink stream processing graphs
- */
-public class StreamGraph extends StreamingPlan {
-
- private static final Logger LOG = LoggerFactory.getLogger(StreamGraph.class);
- private final static String DEAFULT_JOB_NAME = "Flink Streaming Job";
-
- protected boolean chaining = true;
- private String jobName = DEAFULT_JOB_NAME;
-
- // Graph attributes
- private Map<String, Integer> operatorParallelisms;
- private Map<String, Long> bufferTimeouts;
- private Map<String, List<String>> outEdgeLists;
- private Map<String, List<Integer>> outEdgeTypes;
- private Map<String, List<List<String>>> selectedNames;
- private Map<String, List<String>> inEdgeLists;
- private Map<String, List<StreamPartitioner<?>>> outputPartitioners;
- private Map<String, String> operatorNames;
- private Map<String, StreamInvokable<?, ?>> invokableObjects;
- private Map<String, StreamRecordSerializer<?>> typeSerializersIn1;
- private Map<String, StreamRecordSerializer<?>> typeSerializersIn2;
- private Map<String, StreamRecordSerializer<?>> typeSerializersOut1;
- private Map<String, StreamRecordSerializer<?>> typeSerializersOut2;
- private Map<String, Class<? extends AbstractInvokable>> jobVertexClasses;
- private Map<String, List<OutputSelector<?>>> outputSelectors;
- private Map<String, Integer> iterationIds;
- private Map<Integer, String> iterationIDtoHeadName;
- private Map<Integer, String> iterationIDtoTailName;
- private Map<String, Integer> iterationTailCount;
- private Map<String, Long> iterationTimeouts;
- private Map<String, Map<String, OperatorState<?>>> operatorStates;
- private Map<String, InputFormat<String, ?>> inputFormatLists;
-
- private Set<String> sources;
-
- public StreamGraph() {
-
- initGraph();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("StreamGraph created");
- }
- }
-
- public void initGraph() {
- operatorParallelisms = new HashMap<String, Integer>();
- bufferTimeouts = new HashMap<String, Long>();
- outEdgeLists = new HashMap<String, List<String>>();
- outEdgeTypes = new HashMap<String, List<Integer>>();
- selectedNames = new HashMap<String, List<List<String>>>();
- inEdgeLists = new HashMap<String, List<String>>();
- outputPartitioners = new HashMap<String, List<StreamPartitioner<?>>>();
- operatorNames = new HashMap<String, String>();
- invokableObjects = new HashMap<String, StreamInvokable<?, ?>>();
- typeSerializersIn1 = new HashMap<String, StreamRecordSerializer<?>>();
- typeSerializersIn2 = new HashMap<String, StreamRecordSerializer<?>>();
- typeSerializersOut1 = new HashMap<String, StreamRecordSerializer<?>>();
- typeSerializersOut2 = new HashMap<String, StreamRecordSerializer<?>>();
- outputSelectors = new HashMap<String, List<OutputSelector<?>>>();
- jobVertexClasses = new HashMap<String, Class<? extends AbstractInvokable>>();
- iterationIds = new HashMap<String, Integer>();
- iterationIDtoHeadName = new HashMap<Integer, String>();
- iterationIDtoTailName = new HashMap<Integer, String>();
- iterationTailCount = new HashMap<String, Integer>();
- iterationTimeouts = new HashMap<String, Long>();
- operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
- inputFormatLists = new HashMap<String, InputFormat<String, ?>>();
- sources = new HashSet<String>();
- }
-
- /**
- * Adds a vertex to the streaming graph with the given parameters
- *
- * @param vertexName
- * Name of the vertex
- * @param invokableObject
- * User defined operator
- * @param inTypeInfo
- * Input type for serialization
- * @param outTypeInfo
- * Output type for serialization
- * @param operatorName
- * Operator type
- * @param parallelism
- * Number of parallel instances created
- */
- public <IN, OUT> void addStreamVertex(String vertexName,
- StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
- TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
-
- addVertex(vertexName, StreamVertex.class, invokableObject, operatorName, parallelism);
-
- StreamRecordSerializer<IN> inSerializer = inTypeInfo != null ? new StreamRecordSerializer<IN>(
- inTypeInfo) : null;
- StreamRecordSerializer<OUT> outSerializer = outTypeInfo != null ? new StreamRecordSerializer<OUT>(
- outTypeInfo) : null;
-
- addTypeSerializers(vertexName, inSerializer, null, outSerializer, null);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Vertex: {}", vertexName);
- }
- }
-
- public <IN, OUT> void addSourceVertex(String vertexName,
- StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
- TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
- addStreamVertex(vertexName, invokableObject, inTypeInfo, outTypeInfo, operatorName,
- parallelism);
- sources.add(vertexName);
- }
-
- /**
- * Adds a vertex for the iteration head to the {@link JobGraph}. The
- * iterated values will be fed from this vertex back to the graph.
- *
- * @param vertexName
- * Name of the vertex
- * @param iterationHead
- * Id of the iteration head
- * @param iterationID
- * ID of iteration for multiple iterations
- * @param parallelism
- * Number of parallel instances created
- * @param waitTime
- * Max wait time for next record
- */
- public void addIterationHead(String vertexName, String iterationHead, Integer iterationID,
- int parallelism, long waitTime) {
-
- addVertex(vertexName, StreamIterationHead.class, null, null, parallelism);
-
- chaining = false;
-
- iterationIds.put(vertexName, iterationID);
- iterationIDtoHeadName.put(iterationID, vertexName);
-
- setSerializersFrom(iterationHead, vertexName);
-
- setEdge(vertexName, iterationHead,
- outputPartitioners.get(inEdgeLists.get(iterationHead).get(0)).get(0), 0,
- new ArrayList<String>());
-
- iterationTimeouts.put(iterationIDtoHeadName.get(iterationID), waitTime);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("ITERATION SOURCE: {}", vertexName);
- }
-
- sources.add(vertexName);
- }
-
- /**
- * Adds a vertex for the iteration tail to the {@link JobGraph}. The values
- * intended to be iterated will be sent to this sink from the iteration
- * head.
- *
- * @param vertexName
- * Name of the vertex
- * @param iterationTail
- * Id of the iteration tail
- * @param iterationID
- * ID of iteration for mulitple iterations
- * @param parallelism
- * Number of parallel instances created
- * @param waitTime
- * Max waiting time for next record
- */
- public void addIterationTail(String vertexName, String iterationTail, Integer iterationID,
- long waitTime) {
-
- if (bufferTimeouts.get(iterationTail) == 0) {
- throw new RuntimeException("Buffer timeout 0 at iteration tail is not supported.");
- }
-
- addVertex(vertexName, StreamIterationTail.class, null, null, getParallelism(iterationTail));
-
- iterationIds.put(vertexName, iterationID);
- iterationIDtoTailName.put(iterationID, vertexName);
-
- setSerializersFrom(iterationTail, vertexName);
- iterationTimeouts.put(iterationIDtoTailName.get(iterationID), waitTime);
-
- setParallelism(iterationIDtoHeadName.get(iterationID), getParallelism(iterationTail));
- setBufferTimeout(iterationIDtoHeadName.get(iterationID), bufferTimeouts.get(iterationTail));
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("ITERATION SINK: {}", vertexName);
- }
-
- }
-
- public <IN1, IN2, OUT> void addCoTask(String vertexName,
- CoInvokable<IN1, IN2, OUT> taskInvokableObject, TypeInformation<IN1> in1TypeInfo,
- TypeInformation<IN2> in2TypeInfo, TypeInformation<OUT> outTypeInfo,
- String operatorName, int parallelism) {
-
- addVertex(vertexName, CoStreamVertex.class, taskInvokableObject, operatorName, parallelism);
-
- addTypeSerializers(vertexName, new StreamRecordSerializer<IN1>(in1TypeInfo),
- new StreamRecordSerializer<IN2>(in2TypeInfo), new StreamRecordSerializer<OUT>(
- outTypeInfo), null);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("CO-TASK: {}", vertexName);
- }
- }
-
- /**
- * Sets vertex parameters in the JobGraph
- *
- * @param vertexName
- * Name of the vertex
- * @param vertexClass
- * The class of the vertex
- * @param invokableObjectject
- * The user defined invokable object
- * @param operatorName
- * Type of the user defined operator
- * @param parallelism
- * Number of parallel instances created
- */
- private void addVertex(String vertexName, Class<? extends AbstractInvokable> vertexClass,
- StreamInvokable<?, ?> invokableObject, String operatorName, int parallelism) {
-
- jobVertexClasses.put(vertexName, vertexClass);
- setParallelism(vertexName, parallelism);
- invokableObjects.put(vertexName, invokableObject);
- operatorNames.put(vertexName, operatorName);
- outEdgeLists.put(vertexName, new ArrayList<String>());
- outEdgeTypes.put(vertexName, new ArrayList<Integer>());
- selectedNames.put(vertexName, new ArrayList<List<String>>());
- outputSelectors.put(vertexName, new ArrayList<OutputSelector<?>>());
- inEdgeLists.put(vertexName, new ArrayList<String>());
- outputPartitioners.put(vertexName, new ArrayList<StreamPartitioner<?>>());
- iterationTailCount.put(vertexName, 0);
- }
-
- /**
- * Connects two vertices in the JobGraph using the selected partitioner
- * settings
- *
- * @param upStreamVertexName
- * Name of the upstream(output) vertex
- * @param downStreamVertexName
- * Name of the downstream(input) vertex
- * @param partitionerObject
- * Partitioner object
- * @param typeNumber
- * Number of the type (used at co-functions)
- * @param outputNames
- * User defined names of the out edge
- */
- public void setEdge(String upStreamVertexName, String downStreamVertexName,
- StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames) {
- outEdgeLists.get(upStreamVertexName).add(downStreamVertexName);
- outEdgeTypes.get(upStreamVertexName).add(typeNumber);
- inEdgeLists.get(downStreamVertexName).add(upStreamVertexName);
- outputPartitioners.get(upStreamVertexName).add(partitionerObject);
- selectedNames.get(upStreamVertexName).add(outputNames);
- }
-
- private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
- StreamRecordSerializer<?> in2, StreamRecordSerializer<?> out1,
- StreamRecordSerializer<?> out2) {
- typeSerializersIn1.put(vertexName, in1);
- typeSerializersIn2.put(vertexName, in2);
- typeSerializersOut1.put(vertexName, out1);
- typeSerializersOut2.put(vertexName, out2);
- }
-
- /**
- * Sets the number of parallel instances created for the given vertex.
- *
- * @param vertexName
- * Name of the vertex
- * @param parallelism
- * Number of parallel instances created
- */
- public void setParallelism(String vertexName, int parallelism) {
- operatorParallelisms.put(vertexName, parallelism);
- }
-
- public int getParallelism(String vertexName) {
- return operatorParallelisms.get(vertexName);
- }
-
- /**
- * Sets the input format for the given vertex.
- *
- * @param vertexName
- * Name of the vertex
- * @param inputFormat
- * input format of the file source associated with the given
- * vertex
- */
- public void setInputFormat(String vertexName, InputFormat<String, ?> inputFormat) {
- inputFormatLists.put(vertexName, inputFormat);
- }
-
- public void setBufferTimeout(String vertexName, long bufferTimeout) {
- this.bufferTimeouts.put(vertexName, bufferTimeout);
- }
-
- public long getBufferTimeout(String vertexName) {
- return this.bufferTimeouts.get(vertexName);
- }
-
- public void addOperatorState(String veretxName, String stateName, OperatorState<?> state) {
- Map<String, OperatorState<?>> states = operatorStates.get(veretxName);
- if (states == null) {
- states = new HashMap<String, OperatorState<?>>();
- states.put(stateName, state);
- } else {
- if (states.containsKey(stateName)) {
- throw new RuntimeException("State has already been registered with this name: "
- + stateName);
- } else {
- states.put(stateName, state);
- }
- }
- operatorStates.put(veretxName, states);
- }
-
- /**
- * Sets a user defined {@link OutputSelector} for the given operator. Used
- * for directed emits.
- *
- * @param vertexName
- * Name of the vertex for which the output selector will be set
- * @param outputSelector
- * The user defined output selector.
- */
- public <T> void setOutputSelector(String vertexName, OutputSelector<T> outputSelector) {
- outputSelectors.get(vertexName).add(outputSelector);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Outputselector set for {}", vertexName);
- }
-
- }
-
- public <IN, OUT> void setInvokable(String vertexName, StreamInvokable<IN, OUT> invokableObject) {
- invokableObjects.put(vertexName, invokableObject);
- }
-
- public <OUT> void setOutType(String id, TypeInformation<OUT> outType) {
- StreamRecordSerializer<OUT> serializer = new StreamRecordSerializer<OUT>(outType);
- typeSerializersOut1.put(id, serializer);
- }
-
- public StreamInvokable<?, ?> getInvokable(String vertexName) {
- return invokableObjects.get(vertexName);
- }
-
- @SuppressWarnings("unchecked")
- public <OUT> StreamRecordSerializer<OUT> getOutSerializer1(String vertexName) {
- return (StreamRecordSerializer<OUT>) typeSerializersOut1.get(vertexName);
- }
-
- @SuppressWarnings("unchecked")
- public <OUT> StreamRecordSerializer<OUT> getOutSerializer2(String vertexName) {
- return (StreamRecordSerializer<OUT>) typeSerializersOut2.get(vertexName);
- }
-
- @SuppressWarnings("unchecked")
- public <IN> StreamRecordSerializer<IN> getInSerializer1(String vertexName) {
- return (StreamRecordSerializer<IN>) typeSerializersIn1.get(vertexName);
- }
-
- @SuppressWarnings("unchecked")
- public <IN> StreamRecordSerializer<IN> getInSerializer2(String vertexName) {
- return (StreamRecordSerializer<IN>) typeSerializersIn2.get(vertexName);
- }
-
- /**
- * Sets TypeSerializerWrapper from one vertex to another, used with some
- * sinks.
- *
- * @param from
- * from
- * @param to
- * to
- */
- public void setSerializersFrom(String from, String to) {
- operatorNames.put(to, operatorNames.get(from));
-
- typeSerializersIn1.put(to, typeSerializersOut1.get(from));
- typeSerializersIn2.put(to, typeSerializersOut2.get(from));
- typeSerializersOut1.put(to, typeSerializersOut1.get(from));
- typeSerializersOut2.put(to, typeSerializersOut2.get(from));
- }
-
- /**
- * Gets the assembled {@link JobGraph} and adds a default name for it.
- */
- public JobGraph getJobGraph() {
- return getJobGraph(jobName);
- }
-
- /**
- * Gets the assembled {@link JobGraph} and adds a user specified name for
- * it.
- *
- * @param jobGraphName
- * name of the jobGraph
- */
- public JobGraph getJobGraph(String jobGraphName) {
-
- this.jobName = jobGraphName;
- StreamingJobGraphGenerator optimizer = new StreamingJobGraphGenerator(this);
-
- return optimizer.createJobGraph(jobGraphName);
- }
-
- public void setJobName(String jobName) {
- this.jobName = jobName;
- }
-
- public void setChaining(boolean chaining) {
- this.chaining = chaining;
- }
-
- public Collection<String> getSources() {
- return sources;
- }
-
- public List<String> getOutEdges(String vertexName) {
- return outEdgeLists.get(vertexName);
- }
-
- public List<String> getInEdges(String vertexName) {
- return inEdgeLists.get(vertexName);
- }
-
- public List<Integer> getOutEdgeTypes(String vertexName) {
-
- return outEdgeTypes.get(vertexName);
- }
-
- public StreamPartitioner<?> getOutPartitioner(String upStreamVertex, String downStreamVertex) {
- return outputPartitioners.get(upStreamVertex).get(
- outEdgeLists.get(upStreamVertex).indexOf(downStreamVertex));
- }
-
- public List<String> getSelectedNames(String upStreamVertex, String downStreamVertex) {
-
- return selectedNames.get(upStreamVertex).get(
- outEdgeLists.get(upStreamVertex).indexOf(downStreamVertex));
- }
-
- public Collection<Integer> getIterationIDs() {
- return new HashSet<Integer>(iterationIds.values());
- }
-
- public String getIterationTail(int iterID) {
- return iterationIDtoTailName.get(iterID);
- }
-
- public String getIterationHead(int iterID) {
- return iterationIDtoHeadName.get(iterID);
- }
-
- public Class<? extends AbstractInvokable> getJobVertexClass(String vertexName) {
- return jobVertexClasses.get(vertexName);
- }
-
- public InputFormat<String, ?> getInputFormat(String vertexName) {
- return inputFormatLists.get(vertexName);
- }
-
- public List<OutputSelector<?>> getOutputSelector(String vertexName) {
- return outputSelectors.get(vertexName);
- }
-
- public Map<String, OperatorState<?>> getState(String vertexName) {
- return operatorStates.get(vertexName);
- }
-
- public Integer getIterationID(String vertexName) {
- return iterationIds.get(vertexName);
- }
-
- public long getIterationTimeout(String vertexName) {
- return iterationTimeouts.get(vertexName);
- }
-
- public String getOperatorName(String vertexName) {
- return operatorNames.get(vertexName);
- }
-
- @Override
- public String getStreamingPlanAsJSON() {
-
- try {
- JSONObject json = new JSONObject();
- JSONArray nodes = new JSONArray();
-
- json.put("nodes", nodes);
-
- for (String id : operatorNames.keySet()) {
- JSONObject node = new JSONObject();
- nodes.put(node);
-
- node.put("id", Integer.valueOf(id));
- node.put("type", getOperatorName(id));
-
- if (sources.contains(id)) {
- node.put("pact", "Data Source");
- } else {
- node.put("pact", "Data Stream");
- }
-
- node.put("contents", getOperatorName(id) + " at "
- + getInvokable(id).getUserFunction().getClass().getSimpleName());
- node.put("parallelism", getParallelism(id));
-
- int numIn = getInEdges(id).size();
- if (numIn > 0) {
-
- JSONArray inputs = new JSONArray();
- node.put("predecessors", inputs);
-
- for (int i = 0; i < numIn; i++) {
-
- String inID = getInEdges(id).get(i);
-
- JSONObject input = new JSONObject();
- inputs.put(input);
-
- input.put("id", Integer.valueOf(inID));
- input.put("ship_strategy", getOutPartitioner(inID, id).getStrategy());
- if (i == 0) {
- input.put("side", "first");
- } else if (i == 1) {
- input.put("side", "second");
- }
- }
- }
-
- }
- return json.toString();
- } catch (JSONException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("JSON plan creation failed: {}", e);
- }
- return "";
- }
-
- }
-
- @Override
- public void dumpStreamingPlanAsJSON(File file) throws IOException {
- PrintWriter pw = null;
- try {
- pw = new PrintWriter(new FileOutputStream(file), false);
- pw.write(getStreamingPlanAsJSON());
- pw.flush();
-
- } finally {
- if (pw != null) {
- pw.close();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
deleted file mode 100644
index 24ebf46..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
-import org.apache.flink.runtime.jobgraph.DistributionPattern;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.ScheduleMode;
-import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
-import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
-import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
-import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
-import org.apache.flink.streaming.partitioner.StreamPartitioner;
-import org.apache.flink.streaming.partitioner.StreamPartitioner.PartitioningStrategy;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class StreamingJobGraphGenerator {
-
- private static final Logger LOG = LoggerFactory.getLogger(StreamingJobGraphGenerator.class);
-
- private StreamGraph streamGraph;
-
- private Map<String, AbstractJobVertex> streamVertices;
- private JobGraph jobGraph;
- private Collection<String> builtNodes;
-
- private Map<String, Map<String, StreamConfig>> chainedConfigs;
- private Map<String, StreamConfig> vertexConfigs;
- private Map<String, String> chainedNames;
-
- public StreamingJobGraphGenerator(StreamGraph streamGraph) {
- this.streamGraph = streamGraph;
- }
-
- private void init() {
- this.streamVertices = new HashMap<String, AbstractJobVertex>();
- this.builtNodes = new HashSet<String>();
- this.chainedConfigs = new HashMap<String, Map<String, StreamConfig>>();
- this.vertexConfigs = new HashMap<String, StreamConfig>();
- this.chainedNames = new HashMap<String, String>();
- }
-
- public JobGraph createJobGraph(String jobName) {
- jobGraph = new JobGraph(jobName);
- // Turn lazy scheduling off
- jobGraph.setScheduleMode(ScheduleMode.ALL);
-
- init();
-
- for (String sourceName : streamGraph.getSources()) {
- createChain(sourceName, sourceName);
- }
-
- setSlotSharing();
-
- return jobGraph;
- }
-
- private List<Tuple2<String, String>> createChain(String startNode, String current) {
-
- if (!builtNodes.contains(startNode)) {
-
- List<Tuple2<String, String>> transitiveOutEdges = new ArrayList<Tuple2<String, String>>();
- List<String> chainableOutputs = new ArrayList<String>();
- List<String> nonChainableOutputs = new ArrayList<String>();
-
- for (String outName : streamGraph.getOutEdges(current)) {
- if (isChainable(current, outName)) {
- chainableOutputs.add(outName);
- } else {
- nonChainableOutputs.add(outName);
- }
- }
-
- for (String chainable : chainableOutputs) {
- transitiveOutEdges.addAll(createChain(startNode, chainable));
- }
-
- for (String nonChainable : nonChainableOutputs) {
- transitiveOutEdges.add(new Tuple2<String, String>(current, nonChainable));
- createChain(nonChainable, nonChainable);
- }
-
- chainedNames.put(current, createChainedName(current, chainableOutputs));
-
- StreamConfig config = current.equals(startNode) ? createProcessingVertex(startNode)
- : new StreamConfig(new Configuration());
-
- setVertexConfig(current, config, chainableOutputs, nonChainableOutputs);
-
- if (current.equals(startNode)) {
-
- config.setChainStart();
- config.setOutEdgesInOrder(transitiveOutEdges);
-
- for (Tuple2<String, String> edge : transitiveOutEdges) {
- connect(startNode, edge);
- }
-
- config.setTransitiveChainedTaskConfigs(chainedConfigs.get(startNode));
-
- } else {
-
- Map<String, StreamConfig> chainedConfs = chainedConfigs.get(startNode);
-
- if (chainedConfs == null) {
- chainedConfigs.put(startNode, new HashMap<String, StreamConfig>());
- }
- chainedConfigs.get(startNode).put(current, config);
- }
-
- return transitiveOutEdges;
-
- } else {
- return new ArrayList<Tuple2<String, String>>();
- }
- }
-
- private String createChainedName(String vertexID, List<String> chainedOutputs) {
- String vertexName = streamGraph.getOperatorName(vertexID);
- if (chainedOutputs.size() > 1) {
- List<String> outputChainedNames = new ArrayList<String>();
- for (String chainable : chainedOutputs) {
- outputChainedNames.add(chainedNames.get(chainable));
- }
- return vertexName + " -> (" + StringUtils.join(outputChainedNames, ", ") + ")";
- } else if (chainedOutputs.size() == 1) {
- return vertexName + " -> " + chainedNames.get(chainedOutputs.get(0));
- } else {
- return vertexName;
- }
-
- }
-
- private StreamConfig createProcessingVertex(String vertexName) {
-
- AbstractJobVertex vertex = new AbstractJobVertex(chainedNames.get(vertexName));
-
- vertex.setInvokableClass(streamGraph.getJobVertexClass(vertexName));
- if (streamGraph.getParallelism(vertexName) > 0) {
- vertex.setParallelism(streamGraph.getParallelism(vertexName));
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Parallelism set: {} for {}", streamGraph.getParallelism(vertexName),
- vertexName);
- }
-
- if (streamGraph.getInputFormat(vertexName) != null) {
- vertex.setInputSplitSource(streamGraph.getInputFormat(vertexName));
- }
-
- streamVertices.put(vertexName, vertex);
- builtNodes.add(vertexName);
- jobGraph.addVertex(vertex);
-
- return new StreamConfig(vertex.getConfiguration());
- }
-
- private void setVertexConfig(String vertexName, StreamConfig config,
- List<String> chainableOutputs, List<String> nonChainableOutputs) {
-
- config.setVertexName(vertexName);
- config.setBufferTimeout(streamGraph.getBufferTimeout(vertexName));
-
- config.setTypeSerializerIn1(streamGraph.getInSerializer1(vertexName));
- config.setTypeSerializerIn2(streamGraph.getInSerializer2(vertexName));
- config.setTypeSerializerOut1(streamGraph.getOutSerializer1(vertexName));
- config.setTypeSerializerOut2(streamGraph.getOutSerializer2(vertexName));
-
- config.setUserInvokable(streamGraph.getInvokable(vertexName));
- config.setOutputSelectors(streamGraph.getOutputSelector(vertexName));
- config.setOperatorStates(streamGraph.getState(vertexName));
-
- config.setNumberOfOutputs(nonChainableOutputs.size());
- config.setOutputs(nonChainableOutputs);
- config.setChainedOutputs(chainableOutputs);
-
- Class<? extends AbstractInvokable> vertexClass = streamGraph.getJobVertexClass(vertexName);
-
- if (vertexClass.equals(StreamIterationHead.class)
- || vertexClass.equals(StreamIterationTail.class)) {
- config.setIterationId(streamGraph.getIterationID(vertexName));
- config.setIterationWaitTime(streamGraph.getIterationTimeout(vertexName));
- }
-
- List<String> allOutputs = new ArrayList<String>(chainableOutputs);
- allOutputs.addAll(nonChainableOutputs);
-
- for (String output : allOutputs) {
- config.setSelectedNames(output, streamGraph.getSelectedNames(vertexName, output));
- }
-
- vertexConfigs.put(vertexName, config);
- }
-
- private <T> void connect(String headOfChain, Tuple2<String, String> edge) {
-
- String upStreamVertexName = edge.f0;
- String downStreamVertexName = edge.f1;
-
- int outputIndex = streamGraph.getOutEdges(upStreamVertexName).indexOf(downStreamVertexName);
-
- AbstractJobVertex headVertex = streamVertices.get(headOfChain);
- AbstractJobVertex downStreamVertex = streamVertices.get(downStreamVertexName);
-
- StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());
- StreamConfig upStreamConfig = new StreamConfig(headVertex.getConfiguration());
-
- List<Integer> outEdgeIndexList = streamGraph.getOutEdgeTypes(upStreamVertexName);
- int numOfInputs = downStreamConfig.getNumberOfInputs();
-
- downStreamConfig.setInputIndex(numOfInputs++, outEdgeIndexList.get(outputIndex));
- downStreamConfig.setNumberOfInputs(numOfInputs);
-
- StreamPartitioner<?> partitioner = streamGraph.getOutPartitioner(upStreamVertexName,
- downStreamVertexName);
-
- upStreamConfig.setPartitioner(downStreamVertexName, partitioner);
-
- if (partitioner.getStrategy() == PartitioningStrategy.FORWARD) {
- downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
- } else {
- downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.ALL_TO_ALL);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(),
- headOfChain, downStreamVertexName);
- }
- }
-
- private boolean isChainable(String vertexName, String outName) {
-
- StreamInvokable<?, ?> headInvokable = streamGraph.getInvokable(vertexName);
- StreamInvokable<?, ?> outInvokable = streamGraph.getInvokable(outName);
-
- return streamGraph.getInEdges(outName).size() == 1
- && outInvokable != null
- && outInvokable.getChainingStrategy() == ChainingStrategy.ALWAYS
- && (headInvokable.getChainingStrategy() == ChainingStrategy.HEAD || headInvokable
- .getChainingStrategy() == ChainingStrategy.ALWAYS)
- && streamGraph.getOutPartitioner(vertexName, outName).getStrategy() == PartitioningStrategy.FORWARD
- && streamGraph.getParallelism(vertexName) == streamGraph.getParallelism(outName)
- && streamGraph.chaining;
- }
-
- private void setSlotSharing() {
- SlotSharingGroup shareGroup = new SlotSharingGroup();
-
- for (AbstractJobVertex vertex : streamVertices.values()) {
- vertex.setSlotSharingGroup(shareGroup);
- }
-
- for (Integer iterID : streamGraph.getIterationIDs()) {
- CoLocationGroup ccg = new CoLocationGroup();
- AbstractJobVertex tail = streamVertices.get(streamGraph.getIterationTail(iterID));
- AbstractJobVertex head = streamVertices.get(streamGraph.getIterationHead(iterID));
-
- ccg.addVertex(head);
- ccg.addVertex(tail);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
deleted file mode 100644
index 1281bf0..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.util.Collector;
-
-public class CollectorWrapper<OUT> implements Collector<OUT> {
-
- private List<Collector<OUT>> outputs;
-
- public CollectorWrapper() {
- this.outputs = new LinkedList<Collector<OUT>>();
- }
-
- @SuppressWarnings("unchecked")
- public void addCollector(Collector<?> output) {
- outputs.add((Collector<OUT>) output);
- }
-
- @Override
- public void collect(OUT record) {
- for(Collector<OUT> output: outputs){
- output.collect(record);
- }
- }
-
- @Override
- public void close() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
deleted file mode 100755
index 4681cd3..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A StreamCollector that uses user defined output names and a user defined
- * output selector to make directed emits.
- *
- * @param <OUT>
- * Type of the Tuple collected.
- */
-public class DirectedCollectorWrapper<OUT> extends CollectorWrapper<OUT> {
-
- private static final Logger LOG = LoggerFactory.getLogger(DirectedCollectorWrapper.class);
-
- List<OutputSelector<OUT>> outputSelectors;
-
- protected Map<String, List<Collector<OUT>>> outputMap;
-
- private List<Collector<OUT>> selectAllOutputs;
- private Set<Collector<OUT>> emitted;
-
- /**
- * Creates a new DirectedStreamCollector
- *
- * @param outputSelector
- * User defined {@link OutputSelector}
- */
- public DirectedCollectorWrapper(List<OutputSelector<OUT>> outputSelectors) {
- this.outputSelectors = outputSelectors;
- this.emitted = new HashSet<Collector<OUT>>();
- this.selectAllOutputs = new LinkedList<Collector<OUT>>();
- this.outputMap = new HashMap<String, List<Collector<OUT>>>();
-
- }
-
- @Override
- public void addCollector(Collector<?> output) {
- addCollector(output, new ArrayList<String>());
- }
-
- @SuppressWarnings("unchecked")
- public void addCollector(Collector<?> output, List<String> selectedNames) {
-
- if (selectedNames.isEmpty()) {
- selectAllOutputs.add((Collector<OUT>) output);
- } else {
- for (String selectedName : selectedNames) {
-
- if (!outputMap.containsKey(selectedName)) {
- outputMap.put(selectedName, new LinkedList<Collector<OUT>>());
- outputMap.get(selectedName).add((Collector<OUT>) output);
- } else {
- if (!outputMap.get(selectedName).contains(output)) {
- outputMap.get(selectedName).add((Collector<OUT>) output);
- }
- }
-
- }
- }
- }
-
- @Override
- public void collect(OUT record) {
- emitted.clear();
-
- for (Collector<OUT> output : selectAllOutputs) {
- output.collect(record);
- emitted.add(output);
- }
-
- for (OutputSelector<OUT> outputSelector : outputSelectors) {
- Iterable<String> outputNames = outputSelector.select(record);
-
- for (String outputName : outputNames) {
- List<Collector<OUT>> outputList = outputMap.get(outputName);
- if (outputList == null) {
- if (LOG.isErrorEnabled()) {
- String format = String.format(
- "Cannot emit because no output is selected with the name: %s",
- outputName);
- LOG.error(format);
-
- }
- } else {
- for (Collector<OUT> output : outputList) {
- if (!emitted.contains(output)) {
- output.collect(record);
- emitted.add(output);
- }
- }
-
- }
-
- }
- }
-
- }
-
- @Override
- public void close() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/OutputSelector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/OutputSelector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/OutputSelector.java
deleted file mode 100644
index 6dbcff4..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/OutputSelector.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.io.Serializable;
-
-import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
-import org.apache.flink.streaming.api.datastream.SplitDataStream;
-
-/**
- * Interface for defining an OutputSelector for a {@link SplitDataStream} using
- * the {@link SingleOutputStreamOperator#split} call. Every output object of a
- * {@link SplitDataStream} will run through this operator to select outputs.
- *
- * @param <OUT>
- * Type parameter of the split values.
- */
-public interface OutputSelector<OUT> extends Serializable {
- /**
- * Method for selecting output names for the emitted objects when using the
- * {@link SingleOutputStreamOperator#split} method. The values will be
- * emitted only to output names which are contained in the returned
- * iterable.
- *
- * @param value
- * Output object for which the output selection should be made.
- */
- public Iterable<String> select(OUT value);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
deleted file mode 100644
index 4551c5a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.io.IOException;
-
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.io.StreamRecordWriter;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class StreamOutput<OUT> implements Collector<OUT> {
-
- private static final Logger LOG = LoggerFactory.getLogger(StreamOutput.class);
-
- private RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
- private SerializationDelegate<StreamRecord<OUT>> serializationDelegate;
- private StreamRecord<OUT> streamRecord;
- private int channelID;
-
- public StreamOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
- int channelID, SerializationDelegate<StreamRecord<OUT>> serializationDelegate) {
-
- this.serializationDelegate = serializationDelegate;
-
- if (serializationDelegate != null) {
- this.streamRecord = serializationDelegate.getInstance();
- } else {
- throw new RuntimeException("Serializer cannot be null");
- }
- this.channelID = channelID;
- this.output = output;
- }
-
- public RecordWriter<SerializationDelegate<StreamRecord<OUT>>> getRecordWriter() {
- return output;
- }
-
- @Override
- public void collect(OUT record) {
- streamRecord.setObject(record);
- streamRecord.newId(channelID);
- serializationDelegate.setInstance(streamRecord);
-
- try {
- output.emit(serializationDelegate);
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Emit failed due to: {}", StringUtils.stringifyException(e));
- }
- }
- }
-
- @Override
- public void close() {
- if (output instanceof StreamRecordWriter) {
- ((StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>) output).close();
- } else {
- try {
- output.flush();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
deleted file mode 100755
index a6eade8..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ /dev/null
@@ -1,419 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.ClosureCleaner;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.StreamGraph;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
-import org.apache.flink.streaming.api.function.co.CoMapFunction;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.function.co.CoWindowFunction;
-import org.apache.flink.streaming.api.function.co.RichCoMapFunction;
-import org.apache.flink.streaming.api.function.co.RichCoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoFlatMapInvokable;
-import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
-import org.apache.flink.streaming.api.invokable.operator.co.CoMapInvokable;
-import org.apache.flink.streaming.api.invokable.operator.co.CoReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
-import org.apache.flink.streaming.api.windowing.helper.SystemTimestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-
-/**
- * The ConnectedDataStream represents a stream for two different data types. It
- * can be used to apply transformations like {@link CoMapFunction} on two
- * {@link DataStream}s
- *
- * @param <IN1>
- * Type of the first input data steam.
- * @param <IN2>
- * Type of the second input data stream.
- */
-public class ConnectedDataStream<IN1, IN2> {
-
- protected StreamExecutionEnvironment environment;
- protected StreamGraph jobGraphBuilder;
- protected DataStream<IN1> dataStream1;
- protected DataStream<IN2> dataStream2;
-
- protected boolean isGrouped;
- protected KeySelector<IN1, ?> keySelector1;
- protected KeySelector<IN2, ?> keySelector2;
-
- protected ConnectedDataStream(DataStream<IN1> input1, DataStream<IN2> input2) {
- this.jobGraphBuilder = input1.streamGraph;
- this.environment = input1.environment;
- this.dataStream1 = input1.copy();
- this.dataStream2 = input2.copy();
-
- if ((input1 instanceof GroupedDataStream) && (input2 instanceof GroupedDataStream)) {
- this.isGrouped = true;
- this.keySelector1 = ((GroupedDataStream<IN1>) input1).keySelector;
- this.keySelector2 = ((GroupedDataStream<IN2>) input2).keySelector;
- } else {
- this.isGrouped = false;
- this.keySelector1 = null;
- this.keySelector2 = null;
- }
- }
-
- protected ConnectedDataStream(ConnectedDataStream<IN1, IN2> coDataStream) {
- this.jobGraphBuilder = coDataStream.jobGraphBuilder;
- this.environment = coDataStream.environment;
- this.dataStream1 = coDataStream.getFirst();
- this.dataStream2 = coDataStream.getSecond();
- this.isGrouped = coDataStream.isGrouped;
- this.keySelector1 = coDataStream.keySelector1;
- this.keySelector2 = coDataStream.keySelector2;
- }
-
- public <F> F clean(F f) {
- if (getExecutionEnvironment().getConfig().isClosureCleanerEnabled()) {
- ClosureCleaner.clean(f, true);
- }
- ClosureCleaner.ensureSerializable(f);
- return f;
- }
-
- public StreamExecutionEnvironment getExecutionEnvironment() {
- return environment;
- }
-
- /**
- * Returns the first {@link DataStream}.
- *
- * @return The first DataStream.
- */
- public DataStream<IN1> getFirst() {
- return dataStream1.copy();
- }
-
- /**
- * Returns the second {@link DataStream}.
- *
- * @return The second DataStream.
- */
- public DataStream<IN2> getSecond() {
- return dataStream2.copy();
- }
-
- /**
- * Gets the type of the first input
- *
- * @return The type of the first input
- */
- public TypeInformation<IN1> getInputType1() {
- return dataStream1.getType();
- }
-
- /**
- * Gets the type of the second input
- *
- * @return The type of the second input
- */
- public TypeInformation<IN2> getInputType2() {
- return dataStream2.getType();
- }
-
- /**
- * GroupBy operation for connected data stream. Groups the elements of
- * input1 and input2 according to keyPosition1 and keyPosition2. Used for
- * applying function on grouped data streams for example
- * {@link ConnectedDataStream#reduce}
- *
- * @param keyPosition1
- * The field used to compute the hashcode of the elements in the
- * first input stream.
- * @param keyPosition2
- * The field used to compute the hashcode of the elements in the
- * second input stream.
- * @return @return The transformed {@link ConnectedDataStream}
- */
- public ConnectedDataStream<IN1, IN2> groupBy(int keyPosition1, int keyPosition2) {
- return new ConnectedDataStream<IN1, IN2>(dataStream1.groupBy(keyPosition1),
- dataStream2.groupBy(keyPosition2));
- }
-
- /**
- * GroupBy operation for connected data stream. Groups the elements of
- * input1 and input2 according to keyPositions1 and keyPositions2. Used for
- * applying function on grouped data streams for example
- * {@link ConnectedDataStream#reduce}
- *
- * @param keyPositions1
- * The fields used to group the first input stream.
- * @param keyPositions2
- * The fields used to group the second input stream.
- * @return @return The transformed {@link ConnectedDataStream}
- */
- public ConnectedDataStream<IN1, IN2> groupBy(int[] keyPositions1, int[] keyPositions2) {
- return new ConnectedDataStream<IN1, IN2>(dataStream1.groupBy(keyPositions1),
- dataStream2.groupBy(keyPositions2));
- }
-
- /**
- * GroupBy operation for connected data stream using key expressions. Groups
- * the elements of input1 and input2 according to field1 and field2. A field
- * expression is either the name of a public field or a getter method with
- * parentheses of the {@link DataStream}S underlying type. A dot can be used
- * to drill down into objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field1
- * The grouping expression for the first input
- * @param field2
- * The grouping expression for the second input
- * @return The grouped {@link ConnectedDataStream}
- */
- public ConnectedDataStream<IN1, IN2> groupBy(String field1, String field2) {
- return new ConnectedDataStream<IN1, IN2>(dataStream1.groupBy(field1),
- dataStream2.groupBy(field2));
- }
-
- /**
- * GroupBy operation for connected data stream using key expressions. Groups
- * the elements of input1 and input2 according to fields1 and fields2. A
- * field expression is either the name of a public field or a getter method
- * with parentheses of the {@link DataStream}S underlying type. A dot can be
- * used to drill down into objects, as in {@code "field1.getInnerField2()" }
- * .
- *
- * @param fields1
- * The grouping expressions for the first input
- * @param fields2
- * The grouping expressions for the second input
- * @return The grouped {@link ConnectedDataStream}
- */
- public ConnectedDataStream<IN1, IN2> groupBy(String[] fields1, String[] fields2) {
- return new ConnectedDataStream<IN1, IN2>(dataStream1.groupBy(fields1),
- dataStream2.groupBy(fields2));
- }
-
- /**
- * GroupBy operation for connected data stream. Groups the elements of
- * input1 and input2 using keySelector1 and keySelector2. Used for applying
- * function on grouped data streams for example
- * {@link ConnectedDataStream#reduce}
- *
- * @param keySelector1
- * The {@link KeySelector} used for grouping the first input
- * @param keySelector2
- * The {@link KeySelector} used for grouping the second input
- * @return @return The transformed {@link ConnectedDataStream}
- */
- public ConnectedDataStream<IN1, IN2> groupBy(KeySelector<IN1, ?> keySelector1,
- KeySelector<IN2, ?> keySelector2) {
- return new ConnectedDataStream<IN1, IN2>(dataStream1.groupBy(keySelector1),
- dataStream2.groupBy(keySelector2));
- }
-
- /**
- * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
- * the output to a common type. The transformation calls a
- * {@link CoMapFunction#map1} for each element of the first input and
- * {@link CoMapFunction#map2} for each element of the second input. Each
- * CoMapFunction call returns exactly one element. The user can also extend
- * {@link RichCoMapFunction} to gain access to other features provided by
- * the {@link RichFuntion} interface.
- *
- * @param coMapper
- * The CoMapFunction used to jointly transform the two input
- * DataStreams
- * @return The transformed {@link DataStream}
- */
- public <OUT> SingleOutputStreamOperator<OUT, ?> map(CoMapFunction<IN1, IN2, OUT> coMapper) {
- TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoMapFunction.class,
- coMapper.getClass(), 2, null, null);
-
- return addCoFunction("Co-Map", outTypeInfo, new CoMapInvokable<IN1, IN2, OUT>(
- clean(coMapper)));
-
- }
-
- /**
- * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
- * maps the output to a common type. The transformation calls a
- * {@link CoFlatMapFunction#flatMap1} for each element of the first input
- * and {@link CoFlatMapFunction#flatMap2} for each element of the second
- * input. Each CoFlatMapFunction call returns any number of elements
- * including none. The user can also extend {@link RichFlatMapFunction} to
- * gain access to other features provided by the {@link RichFuntion}
- * interface.
- *
- * @param coFlatMapper
- * The CoFlatMapFunction used to jointly transform the two input
- * DataStreams
- * @return The transformed {@link DataStream}
- */
- public <OUT> SingleOutputStreamOperator<OUT, ?> flatMap(
- CoFlatMapFunction<IN1, IN2, OUT> coFlatMapper) {
- TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoFlatMapFunction.class,
- coFlatMapper.getClass(), 2, null, null);
-
- return addCoFunction("Co-Flat Map", outTypeInfo, new CoFlatMapInvokable<IN1, IN2, OUT>(
- clean(coFlatMapper)));
- }
-
- /**
- * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
- * the outputs to a common type. If the {@link ConnectedDataStream} is
- * batched or windowed then the reduce transformation is applied on every
- * sliding batch/window of the data stream. If the connected data stream is
- * grouped then the reducer is applied on every group of elements sharing
- * the same key. This type of reduce is much faster than reduceGroup since
- * the reduce function can be applied incrementally. The user can also
- * extend the {@link RichCoReduceFunction} to gain access to other features
- * provided by the {@link RichFuntion} interface.
- *
- * @param coReducer
- * The {@link CoReduceFunction} that will be called for every
- * element of the inputs.
- * @return The transformed {@link DataStream}.
- */
- public <OUT> SingleOutputStreamOperator<OUT, ?> reduce(CoReduceFunction<IN1, IN2, OUT> coReducer) {
-
- TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoReduceFunction.class,
- coReducer.getClass(), 2, null, null);
-
- return addCoFunction("Co-Reduce", outTypeInfo, getReduceInvokable(clean(coReducer)));
-
- }
-
- /**
- * Applies a CoWindow transformation on the connected DataStreams. The
- * transformation calls the {@link CoWindowFunction#coWindow} method for for
- * time aligned windows of the two data streams. System time is used as
- * default to compute windows.
- *
- * @param coWindowFunction
- * The {@link CoWindowFunction} that will be applied for the time
- * windows.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- *
- * @return The transformed {@link DataStream}.
- */
- @SuppressWarnings("unchecked")
- public <OUT> SingleOutputStreamOperator<OUT, ?> windowReduce(
- CoWindowFunction<IN1, IN2, OUT> coWindowFunction, long windowSize, long slideInterval) {
- return windowReduce(coWindowFunction, windowSize, slideInterval,
- (TimestampWrapper<IN1>) SystemTimestamp.getWrapper(),
- (TimestampWrapper<IN2>) SystemTimestamp.getWrapper());
- }
-
- /**
- * Applies a CoWindow transformation on the connected DataStreams. The
- * transformation calls the {@link CoWindowFunction#coWindow} method for
- * time aligned windows of the two data streams. The user can implement
- * their own time stamps or use the system time by default.
- *
- * @param coWindowFunction
- * The {@link CoWindowFunction} that will be applied for the time
- * windows.
- * @param windowSize
- * Size of the windows that will be aligned for both streams. If
- * system time is used it is milliseconds. User defined time
- * stamps are assumed to be monotonically increasing.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- *
- * @param timestamp1
- * User defined time stamps for the first input.
- * @param timestamp2
- * User defined time stamps for the second input.
- * @return The transformed {@link DataStream}.
- */
- public <OUT> SingleOutputStreamOperator<OUT, ?> windowReduce(
- CoWindowFunction<IN1, IN2, OUT> coWindowFunction, long windowSize, long slideInterval,
- TimestampWrapper<IN1> timestamp1, TimestampWrapper<IN2> timestamp2) {
-
- if (windowSize < 1) {
- throw new IllegalArgumentException("Window size must be positive");
- }
- if (slideInterval < 1) {
- throw new IllegalArgumentException("Slide interval must be positive");
- }
-
- TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoWindowFunction.class,
- coWindowFunction.getClass(), 2, null, null);
-
- return addCoFunction("Co-Window", outTypeInfo, new CoWindowInvokable<IN1, IN2, OUT>(
- clean(coWindowFunction), windowSize, slideInterval, timestamp1, timestamp2));
-
- }
-
- protected <OUT> CoInvokable<IN1, IN2, OUT> getReduceInvokable(
- CoReduceFunction<IN1, IN2, OUT> coReducer) {
- CoReduceInvokable<IN1, IN2, OUT> invokable;
- if (isGrouped) {
- invokable = new CoGroupedReduceInvokable<IN1, IN2, OUT>(clean(coReducer), keySelector1,
- keySelector2);
- } else {
- invokable = new CoReduceInvokable<IN1, IN2, OUT>(clean(coReducer));
- }
- return invokable;
- }
-
- public <OUT> SingleOutputStreamOperator<OUT, ?> addGeneralWindowCombine(
- CoWindowFunction<IN1, IN2, OUT> coWindowFunction, TypeInformation<OUT> outTypeInfo,
- long windowSize, long slideInterval, TimestampWrapper<IN1> timestamp1,
- TimestampWrapper<IN2> timestamp2) {
-
- if (windowSize < 1) {
- throw new IllegalArgumentException("Window size must be positive");
- }
- if (slideInterval < 1) {
- throw new IllegalArgumentException("Slide interval must be positive");
- }
-
- return addCoFunction("Co-Window", outTypeInfo, new CoWindowInvokable<IN1, IN2, OUT>(
- clean(coWindowFunction), windowSize, slideInterval, timestamp1, timestamp2));
-
- }
-
- public <OUT> SingleOutputStreamOperator<OUT, ?> addCoFunction(String functionName,
- TypeInformation<OUT> outTypeInfo, CoInvokable<IN1, IN2, OUT> functionInvokable) {
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- SingleOutputStreamOperator<OUT, ?> returnStream = new SingleOutputStreamOperator(
- environment, functionName, outTypeInfo, functionInvokable);
-
- dataStream1.streamGraph.addCoTask(returnStream.getId(), functionInvokable,
- getInputType1(), getInputType2(), outTypeInfo, functionName,
- environment.getDegreeOfParallelism());
-
- dataStream1.connectGraph(dataStream1, returnStream.getId(), 1);
- dataStream1.connectGraph(dataStream2, returnStream.getId(), 2);
-
- return returnStream;
- }
-
- protected ConnectedDataStream<IN1, IN2> copy() {
- return new ConnectedDataStream<IN1, IN2>(this);
- }
-
-}
[28/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
deleted file mode 100644
index d6a9ac0..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.windowing;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.windowing.deltafunction.EuclideanDistance;
-import org.apache.flink.streaming.api.windowing.extractor.FieldsFromTuple;
-import org.apache.flink.streaming.api.windowing.helper.Count;
-import org.apache.flink.streaming.api.windowing.helper.Delta;
-import org.apache.flink.util.Collector;
-
-/**
- * This example gives an impression about how to use delta policies. It also
- * shows how extractors can be used.
- */
-public class DeltaExtractExample {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- DataStream dstream = env
- .addSource(new CountingSource())
- .window(Delta.of(1.2, new EuclideanDistance(new FieldsFromTuple(0, 1)), new Tuple3(
- 0d, 0d, "foo"))).every(Count.of(2)).reduce(new ConcatStrings());
-
- // emit result
- if (fileOutput) {
- dstream.writeAsText(outputPath, 1);
- } else {
- dstream.print();
- }
-
- // execute the program
- env.execute("Delta Extract Example");
-
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- private static class CountingSource implements SourceFunction<Tuple3<Double, Double, String>> {
- private static final long serialVersionUID = 1L;
-
- private int counter = 0;
-
- @Override
- public void invoke(Collector<Tuple3<Double, Double, String>> collector) throws Exception {
- while (true) {
- if (counter > 9999) {
- counter = 0;
- }
- collector.collect(new Tuple3<Double, Double, String>((double) counter,
- (double) counter + 1, "V" + counter++));
- }
- }
- }
-
- private static final class ConcatStrings implements
- ReduceFunction<Tuple3<Double, Double, String>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple3<Double, Double, String> reduce(Tuple3<Double, Double, String> value1,
- Tuple3<Double, Double, String> value2) throws Exception {
- return new Tuple3<Double, Double, String>(value1.f0, value2.f1, value1.f2 + "|"
- + value2.f2);
- }
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- // parse input arguments
- fileOutput = true;
- if (args.length == 1) {
- outputPath = args[0];
- } else {
- System.err.println("Usage: DeltaExtractExample <result path>");
- return false;
- }
- } else {
- System.out.println("Executing DeltaExtractExample with generated data.");
- System.out.println(" Provide parameter to write to file.");
- System.out.println(" Usage: DeltaExtractExample <result path>");
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
deleted file mode 100644
index 48783f2..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.windowing;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.windowing.helper.Count;
-import org.apache.flink.util.Collector;
-
-/**
- * This example uses count based tumbling windowing with multiple eviction
- * policies at the same time.
- */
-public class MultiplePoliciesExample {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<String> stream = env.addSource(new BasicSource())
- .groupBy(new KeySelector<String, String>(){
- private static final long serialVersionUID = 1L;
- @Override
- public String getKey(String value) throws Exception {
- return value;
- }
- })
- .window(Count.of(2))
- .every(Count.of(3), Count.of(5))
- .reduceGroup(new Concat());
-
- // emit result
- if (fileOutput) {
- stream.writeAsText(outputPath, 1);
- } else {
- stream.print();
- }
-
- // execute the program
- env.execute("Multiple Policies Example");
- }
-
- /**
- * This source function indefinitely provides String inputs for the
- * topology.
- */
- public static final class BasicSource implements SourceFunction<String> {
-
- private static final long serialVersionUID = 1L;
-
- private final static String STR_1 = new String("streaming");
- private final static String STR_2 = new String("flink");
-
- @Override
- public void invoke(Collector<String> out) throws Exception {
- // continuous emit
- while (true) {
- out.collect(STR_1);
- out.collect(STR_2);
- }
- }
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /**
- * This reduce function does a String concat.
- */
- public static final class Concat implements GroupReduceFunction<String, String> {
-
- /**
- * Auto generates version ID
- */
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterable<String> values, Collector<String> out) throws Exception {
- String output = "|";
- for (String v : values) {
- output = output + v + "|";
- }
- out.collect(output);
- }
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- // parse input arguments
- fileOutput = true;
- if (args.length == 1) {
- outputPath = args[0];
- } else {
- System.err.println("Usage: MultiplePoliciesExample <result path>");
- return false;
- }
- } else {
- System.out.println("Executing MultiplePoliciesExample with generated data.");
- System.out.println(" Provide parameter to write to file.");
- System.out.println(" Usage: MultiplePoliciesExample <result path>");
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
deleted file mode 100644
index cf03477..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.windowing;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.windowing.helper.Count;
-import org.apache.flink.util.Collector;
-
-/**
- * This example uses count based sliding windows to illustrate different
- * possibilities for the realization of sliding windows. Take a look on the code
- * which is commented out to see different setups.
- */
-public class SlidingExample {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- /*
- * SIMPLE-EXAMPLE: Use this to always keep the newest 10 elements in the
- * buffer Resulting windows will have an overlap of 5 elements
- */
-
- // DataStream<String> stream = env.addSource(new CountingSource())
- // .window(Count.of(10))
- // .every(Count.of(5))
- // .reduce(new Concat());
-
- /*
- * ADVANCED-EXAMPLE: Use this to have the last element of the last
- * window as first element of the next window while the window size is
- * always 5
- */
-
- DataStream<String> stream = env.addSource(new CountingSource())
- .window(Count.of(5)
- .withDelete(4))
- .every(Count.of(4)
- .startingAt(-1))
- .reduce(new Concat());
-
- // emit result
- if (fileOutput) {
- stream.writeAsText(outputPath, 1);
- } else {
- stream.print();
- }
-
- // execute the program
- env.execute("Sliding Example");
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- private static final class CountingSource implements SourceFunction<String> {
- private static final long serialVersionUID = 1L;
-
- private int counter = 0;
-
- @Override
- public void invoke(Collector<String> collector) throws Exception {
- // continuous emit
- while (true) {
- if (counter > 9999) {
- counter = 0;
- }
- collector.collect("V" + counter++);
- }
- }
- }
-
- /**
- * This reduce function does a String concat.
- */
- private static final class Concat implements ReduceFunction<String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public String reduce(String value1, String value2) throws Exception {
- return value1 + "|" + value2;
- }
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- // parse input arguments
- fileOutput = true;
- if (args.length == 1) {
- outputPath = args[0];
- } else {
- System.err.println("Usage: SlidingExample <result path>");
- return false;
- }
- } else {
- System.out.println("Executing SlidingExample with generated data.");
- System.out.println(" Provide parameter to write to file.");
- System.out.println(" Usage: SlidingExample <result path>");
- }
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
deleted file mode 100644
index 622aa82..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.windowing;
-
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.source.RichSourceFunction;
-import org.apache.flink.streaming.api.windowing.helper.Count;
-import org.apache.flink.streaming.api.windowing.helper.Time;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
-import org.apache.flink.util.Collector;
-
-/**
- * This example shows the functionality of time based windows. It utilizes the
- * {@link ActiveTriggerPolicy} implementation in the
- * {@link ActiveTimeTriggerPolicy}.
- */
-public class TimeWindowingExample {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<Integer> stream = env.addSource(new CountingSourceWithSleep())
- .window(Count.of(100))
- .every(Time.of(1000, TimeUnit.MILLISECONDS))
- .groupBy(new MyKey())
- .sum(0);
-
- // emit result
- if (fileOutput) {
- stream.writeAsText(outputPath, 1);
- } else {
- stream.print();
- }
-
- // execute the program
- env.execute("Time Windowing Example");
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /**
- * This data source emit one element every 0.001 sec. The output is an
- * Integer counting the output elements. As soon as the counter reaches
- * 10000 it is reset to 0. On each reset the source waits 5 sec. before it
- * restarts to produce elements.
- */
- private static final class CountingSourceWithSleep extends RichSourceFunction<Integer> {
- private static final long serialVersionUID = 1L;
-
- private int counter = 0;
- private transient Random rnd;
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- rnd = new Random();
- }
-
- @Override
- public void invoke(Collector<Integer> collector) throws Exception {
- // continuous emit
- while (true) {
- if (counter > 9999) {
- System.out.println("Source pauses now!");
- Thread.sleep(5000);
- System.out.println("Source continouse with emitting now!");
- counter = 0;
- }
- collector.collect(rnd.nextInt(9) + 1);
-
- // Wait 0.001 sec. before the next emit. Otherwise the source is
- // too fast for local tests and you might always see
- // SUM[k=1..9999](k) as result.
- Thread.sleep(1);
- counter++;
- }
- }
- }
-
- private static final class MyKey implements KeySelector<Integer, Integer> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(Integer value) throws Exception {
- if (value < 2) {
- return 0;
- } else {
- return 1;
- }
- }
-
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- // parse input arguments
- fileOutput = true;
- if (args.length == 1) {
- outputPath = args[0];
- } else {
- System.err.println("Usage: TimeWindowingExample <result path>");
- return false;
- }
- } else {
- System.out.println("Executing TimeWindowingExample with generated data.");
- System.out.println(" Provide parameter to write to file.");
- System.out.println(" Usage: TimeWindowingExample <result path>");
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
deleted file mode 100644
index 0f5d8eb..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.windowing;
-
-import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
-import org.apache.flink.streaming.api.windowing.helper.Delta;
-import org.apache.flink.streaming.api.windowing.helper.Time;
-import org.apache.flink.util.Collector;
-
-import java.util.Arrays;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-/**
- * An example of grouped stream windowing where different eviction and trigger
- * policies can be used. A source fetches events from cars every 1 sec
- * containing their id, their current speed (kmh), overall elapsed distance (m)
- * and a timestamp. The streaming example triggers the top speed of each car
- * every x meters elapsed for the last y seconds.
- */
-public class TopSpeedWindowingExample {
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings({ "rawtypes", "serial" })
- DataStream topSpeeds = env
- .addSource(CarSource.create(numOfCars))
- .groupBy(0)
- .window(Time.of(evictionSec, TimeUnit.SECONDS))
- .every(Delta.of(triggerMeters,
- new DeltaFunction<Tuple4<Integer, Integer, Double, Long>>() {
- @Override
- public double getDelta(
- Tuple4<Integer, Integer, Double, Long> oldDataPoint,
- Tuple4<Integer, Integer, Double, Long> newDataPoint) {
- return newDataPoint.f2 - oldDataPoint.f2;
- }
- }, new Tuple4<Integer, Integer, Double, Long>(0, 0, 0d, 0l))).maxBy(1);
-
- topSpeeds.print();
- env.execute("CarTopSpeedWindowingExample");
- }
-
- private static class CarSource implements
- SourceFunction<Tuple4<Integer, Integer, Double, Long>> {
-
- private static final long serialVersionUID = 1L;
- private Integer[] speeds;
- private Double[] distances;
-
- private Random rand = new Random();
-
- private CarSource(int numOfCars) {
- speeds = new Integer[numOfCars];
- distances = new Double[numOfCars];
- Arrays.fill(speeds, 50);
- Arrays.fill(distances, 0d);
- }
-
- public static CarSource create(int cars) {
- return new CarSource(cars);
- }
-
- @Override
- public void invoke(Collector<Tuple4<Integer, Integer, Double, Long>> collector)
- throws Exception {
-
- while (true) {
- Thread.sleep(1000);
- for (int carId = 0; carId < speeds.length; carId++) {
- if (rand.nextBoolean()) {
- speeds[carId] = Math.min(100, speeds[carId] + 5);
- } else {
- speeds[carId] = Math.max(0, speeds[carId] - 5);
- }
- distances[carId] += speeds[carId] / 3.6d;
- collector.collect(new Tuple4<Integer, Integer, Double, Long>(carId,
- speeds[carId], distances[carId], System.currentTimeMillis()));
- }
- }
- }
- }
-
- private static int numOfCars = 2;
- private static int evictionSec = 10;
- private static double triggerMeters = 50;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- if (args.length == 3) {
- numOfCars = Integer.valueOf(args[0]);
- evictionSec = Integer.valueOf(args[1]);
- triggerMeters = Double.valueOf(args[2]);
- } else {
- System.err
- .println("Usage: TopSpeedWindowingExample <numCars> <evictSec> <triggerMeters>");
- return false;
- }
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/PojoExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/PojoExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/PojoExample.java
deleted file mode 100644
index f377863..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/PojoExample.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.examples.wordcount;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.examples.java.wordcount.util.WordCountData;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.Collector;
-
-/**
- * This example shows an implementation of WordCount without using the Tuple2
- * type, but a custom class.
- *
- * <p>
- * Usage: <code>WordCount <text path> <result path></code><br>
- * If no parameters are provided, the program is run with default data from
- * {@link WordCountData}.
- *
- * <p>
- * This example shows how to:
- * <ul>
- * <li>use POJO data types,
- * <li>write a simple Flink program,
- * <li>write and use user-defined functions.
- * </ul>
- */
-public class PojoExample {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- // set up the execution environment
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // get input data
- DataStream<String> text = getTextDataStream(env);
-
- DataStream<Word> counts =
- // split up the lines into Word objects
- text.flatMap(new Tokenizer())
- // group by the field word and sum up the frequency
- .groupBy("word").sum("frequency");
-
- if (fileOutput) {
- counts.writeAsText(outputPath);
- } else {
- counts.print();
- }
-
- // execute program
- env.execute("WordCount Pojo Example");
- }
-
- // *************************************************************************
- // DATA TYPES
- // *************************************************************************
-
- /**
- * This is the POJO (Plain Old Java Object) that is being used for all the
- * operations. As long as all fields are public or have a getter/setter, the
- * system can handle them
- */
- public static class Word {
-
- private String word;
- private Integer frequency;
-
- public Word() {
- }
-
- public Word(String word, int i) {
- this.word = word;
- this.frequency = i;
- }
-
- public String getWord() {
- return word;
- }
-
- public void setWord(String word) {
- this.word = word;
- }
-
- public Integer getFrequency() {
- return frequency;
- }
-
- public void setFrequency(Integer frequency) {
- this.frequency = frequency;
- }
-
- @Override
- public String toString() {
- return "(" + word + ", " + frequency + ")";
- }
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /**
- * Implements the string tokenizer that splits sentences into words as a
- * user-defined FlatMapFunction. The function takes a line (String) and
- * splits it into multiple pairs in the form of "(word,1)" (Tuple2<String,
- * Integer>).
- */
- public static final class Tokenizer implements FlatMapFunction<String, Word> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(String value, Collector<Word> out) {
- // normalize and split the line
- String[] tokens = value.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Word(token, 1));
- }
- }
- }
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String textPath;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- // parse input arguments
- fileOutput = true;
- if (args.length == 2) {
- textPath = args[0];
- outputPath = args[1];
- } else {
- System.err.println("Usage: PojoExample <text path> <result path>");
- return false;
- }
- } else {
- System.out.println("Executing PojoExample example with built-in default data.");
- System.out.println(" Provide parameters to read input data from a file.");
- System.out.println(" Usage: PojoExample <text path> <result path>");
- }
- return true;
- }
-
- private static DataStream<String> getTextDataStream(StreamExecutionEnvironment env) {
- if (fileOutput) {
- // read the text file from given input path
- return env.readTextFile(textPath);
- } else {
- // get default test text data
- return env.fromElements(WordCountData.WORDS);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/WordCount.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/WordCount.java
deleted file mode 100644
index 085fe5f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/wordcount/WordCount.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.wordcount;
-
-import java.util.StringTokenizer;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.examples.java.wordcount.util.WordCountData;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.Collector;
-
-/**
- * Implements the "WordCount" program that computes a simple word occurrence
- * histogram over text files in a streaming fashion.
- *
- * <p>
- * The input is a plain text file with lines separated by newline characters.
- *
- * <p>
- * Usage: <code>WordCount <text path> <result path></code><br>
- * If no parameters are provided, the program is run with default data from
- * {@link WordCountData}.
- *
- * <p>
- * This example shows how to:
- * <ul>
- * <li>write a simple Flink Streaming program,
- * <li>use tuple data types,
- * <li>write and use user-defined functions.
- * </ul>
- *
- */
-public class WordCount {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- // set up the execution environment
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- // get input data
- DataStream<String> text = getTextDataStream(env);
-
- DataStream<Tuple2<String, Integer>> counts =
- // split up the lines in pairs (2-tuples) containing: (word,1)
- text.flatMap(new Tokenizer())
- // group by the tuple field "0" and sum up tuple field "1"
- .groupBy(0).sum(1);
-
- // emit result
- if (fileOutput) {
- counts.writeAsText(outputPath, 1);
- } else {
- counts.print();
- }
-
- // execute program
- env.execute("Streaming WordCount");
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /**
- * Implements the string tokenizer that splits sentences into words as a
- * user-defined FlatMapFunction. The function takes a line (String) and
- * splits it into multiple pairs in the form of "(word,1)" (Tuple2<String,
- * Integer>).
- */
- public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(String inTuple, Collector<Tuple2<String, Integer>> out)
- throws Exception {
- // tokenize the line
- StringTokenizer tokenizer = new StringTokenizer(inTuple);
-
- // emit the pairs
- while (tokenizer.hasMoreTokens()) {
- out.collect(new Tuple2<String, Integer>(tokenizer.nextToken(), 1));
- }
- }
- }
-
- // *************************************************************************
- // UTIL METHODS
- // *************************************************************************
-
- private static boolean fileOutput = false;
- private static String textPath;
- private static String outputPath;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- // parse input arguments
- fileOutput = true;
- if (args.length == 2) {
- textPath = args[0];
- outputPath = args[1];
- } else {
- System.err.println("Usage: WordCount <text path> <result path>");
- return false;
- }
- } else {
- System.out.println("Executing WordCount example with built-in default data.");
- System.out.println(" Provide parameters to read input data from a file.");
- System.out.println(" Usage: WordCount <text path> <result path>");
- }
- return true;
- }
-
- private static DataStream<String> getTextDataStream(StreamExecutionEnvironment env) {
- if (fileOutput) {
- // read the text file from given input path
- return env.readTextFile(textPath);
- } else {
- // get default test text data
- return env.fromElements(WordCountData.WORDS);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala
deleted file mode 100644
index b38764c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.scala.examples.socket
-
-import org.apache.flink.streaming.api.scala._
-
-/**
- * This example shows an implementation of WordCount with data from a text socket.
- * To run the example make sure that the service providing the text data is already up and running.
- *
- * To start an example socket text stream on your local machine run netcat from a command line,
- * where the parameter specifies the port number:
- *
- * {{{
- * nc -lk 9999
- * }}}
- *
- * Usage:
- * {{{
- * SocketTextStreamWordCount <hostname> <port> <output path>
- * }}}
- *
- * This example shows how to:
- *
- * - use StreamExecutionEnvironment.socketTextStream
- * - write a simple Flink Streaming program in scala.
- * - write and use user-defined functions.
- */
-object SocketTextStreamWordCount {
-
- def main(args: Array[String]) {
- if (!parseParameters(args)) {
- return
- }
-
- val env = StreamExecutionEnvironment.getExecutionEnvironment
-
- //Create streams for names and ages by mapping the inputs to the corresponding objects
- val text = env.socketTextStream(hostName, port)
- val counts = text.flatMap { _.toLowerCase.split("\\W+") filter { _.nonEmpty } }
- .map { (_, 1) }
- .groupBy(0)
- .sum(1)
-
- if (fileOutput) {
- counts.writeAsCsv(outputPath, 1)
- } else {
- counts print
- }
-
- env.execute("Scala SocketTextStreamWordCount Example")
- }
-
- private def parseParameters(args: Array[String]): Boolean = {
- if (args.length == 3) {
- fileOutput = true
- hostName = args(0)
- port = args(1).toInt
- outputPath = args(2)
- } else if (args.length == 2) {
- hostName = args(0)
- port = args(1).toInt
- } else {
- System.err.println("Usage: SocketTextStreamWordCount <hostname> <port> [<output path>]")
- return false
- }
- true
- }
-
- private var fileOutput: Boolean = false
- private var hostName: String = null
- private var port: Int = 0
- private var outputPath: String = null
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
deleted file mode 100644
index 1091aa3..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.scala.examples.windowing
-
-import java.util.concurrent.TimeUnit._
-
-import org.apache.flink.streaming.api.scala._
-import org.apache.flink.streaming.api.scala.windowing.{Delta, Time}
-
-import scala.Stream._
-import scala.math._
-import scala.util.Random
-
-/**
- * An example of grouped stream windowing where different eviction and
- * trigger policies can be used. A source fetches events from cars
- * every 1 sec containing their id, their current speed (kmh),
- * overall elapsed distance (m) and a timestamp. The streaming
- * example triggers the top speed of each car every x meters elapsed
- * for the last y seconds.
- */
-object TopSpeedWindowing {
-
- case class CarEvent(carId: Int, speed: Int, distance: Double, time: Long) extends Serializable
-
- def main(args: Array[String]) {
- if (!parseParameters(args)) {
- return
- }
-
- val cars = genCarStream().groupBy("carId")
- .window(Time.of(evictionSec, SECONDS))
- .every(Delta.of[CarEvent](triggerMeters,
- (oldSp,newSp) => newSp.distance-oldSp.distance, CarEvent(0,0,0,0)))
- .maxBy("speed")
-
- cars print
-
- StreamExecutionEnvironment.getExecutionEnvironment.execute("TopSpeedWindowing")
-
- }
-
- def genCarStream(): DataStream[CarEvent] = {
-
- def nextSpeed(carEvent : CarEvent) : CarEvent =
- {
- val next =
- if (Random.nextBoolean) min(100, carEvent.speed + 5) else max(0, carEvent.speed - 5)
- CarEvent(carEvent.carId, next, carEvent.distance + next/3.6d,System.currentTimeMillis)
- }
- def carStream(speeds : Stream[CarEvent]) : Stream[CarEvent] =
- {
- Thread.sleep(1000)
- speeds.append(carStream(speeds.map(nextSpeed)))
- }
- carStream(range(0, numOfCars).map(CarEvent(_,50,0,System.currentTimeMillis())))
- }
-
- def parseParameters(args: Array[String]): Boolean = {
- if (args.length > 0) {
- if (args.length == 3) {
- numOfCars = args(0).toInt
- evictionSec = args(1).toInt
- triggerMeters = args(2).toDouble
- }
- else {
- System.err.println("Usage: TopSpeedWindowing <numCars> <evictSec> <triggerMeters>")
- false
- }
- }
- true
- }
-
- var numOfCars = 2
- var evictionSec = 10
- var triggerMeters = 50d
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
deleted file mode 100644
index 0b78365..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.scala.examples.windowing
-
-import org.apache.flink.streaming.api.scala._
-
-import scala.Stream._
-import scala.util.Random
-import java.util.concurrent.TimeUnit
-
-object WindowJoin {
-
- case class Name(id: Long, name: String)
- case class Age(id: Long, age: Int)
- case class Person(name: String, age: Long)
-
- def main(args: Array[String]) {
-
- val env = StreamExecutionEnvironment.getExecutionEnvironment
-
- //Create streams for names and ages by mapping the inputs to the corresponding objects
- val names = env.fromCollection(nameStream).map(x => Name(x._1, x._2))
- val ages = env.fromCollection(ageStream).map(x => Age(x._1, x._2))
-
- //Join the two input streams by id on the last 2 seconds every second and create new
- //Person objects containing both name and age
- val joined =
- names.join(ages).onWindow(2, TimeUnit.SECONDS)
- .every(1, TimeUnit.SECONDS)
- .where("id")
- .equalTo("id") { (n, a) => Person(n.name, a.age) }
-
- joined print
-
- env.execute("WindowJoin")
- }
-
- def nameStream() : Stream[(Long,String)] = {
- def nameMapper(names: Array[String])(x: Int) : (Long, String) =
- {
- if(x%100==0) Thread.sleep(1000)
- (x, names(Random.nextInt(names.length)))
- }
- range(1,10000).map(nameMapper(Array("tom", "jerry", "alice", "bob", "john", "grace")))
- }
-
- def ageStream() : Stream[(Long,Int)] = {
- def ageMapper(x: Int) : (Long, Int) =
- {
- if(x%100==0) Thread.sleep(1000)
- (x, Random.nextInt(90))
- }
- range(1,10000).map(ageMapper)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/pom.xml b/flink-addons/flink-streaming/flink-streaming-scala/pom.xml
deleted file mode 100644
index c06fba7..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/pom.xml
+++ /dev/null
@@ -1,217 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-parent</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-streaming-scala</artifactId>
- <name>flink-streaming-scala</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-scala</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-reflect</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-compiler</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.scalamacros</groupId>
- <artifactId>quasiquotes_${scala.binary.version}</artifactId>
- <version>${scala.macros.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </dependency>
-
- <!-- guava needs to be in "provided" scope, to make sure it is not included into the jars by the shading -->
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- <scope>provided</scope>
- </dependency>
-
- </dependencies>
-
- <build>
- <plugins>
- <!-- Scala Compiler -->
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>3.1.4</version>
- <executions>
- <!-- Run scala compiler in the process-resources phase, so that dependencies on
- scala classes can be resolved later in the (Java) compile phase -->
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
-
- <!-- Run scala compiler in the process-test-resources phase, so that dependencies on
- scala classes can be resolved later in the (Java) test-compile phase -->
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <jvmArgs>
- <jvmArg>-Xms128m</jvmArg>
- <jvmArg>-Xmx512m</jvmArg>
- </jvmArgs>
- <compilerPlugins>
- <compilerPlugin>
- <groupId>org.scalamacros</groupId>
- <artifactId>paradise_${scala.version}</artifactId>
- <version>${scala.macros.version}</version>
- </compilerPlugin>
- </compilerPlugins>
- </configuration>
- </plugin>
-
- <!-- Eclipse Integration -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.8</version>
- <configuration>
- <downloadSources>true</downloadSources>
- <projectnatures>
- <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
- <projectnature>org.eclipse.jdt.core.javanature</projectnature>
- </projectnatures>
- <buildcommands>
- <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
- </buildcommands>
- <classpathContainers>
- <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
- <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
- </classpathContainers>
- <excludes>
- <exclude>org.scala-lang:scala-library</exclude>
- <exclude>org.scala-lang:scala-compiler</exclude>
- </excludes>
- <sourceIncludes>
- <sourceInclude>**/*.scala</sourceInclude>
- <sourceInclude>**/*.java</sourceInclude>
- </sourceIncludes>
- </configuration>
- </plugin>
-
- <!-- Adding scala source directories to build path -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <!-- Add src/main/scala to eclipse build path -->
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/main/scala</source>
- </sources>
- </configuration>
- </execution>
- <!-- Add src/test/scala to eclipse build path -->
- <execution>
- <id>add-test-source</id>
- <phase>generate-test-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/test/scala</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <version>0.5.0</version>
- <executions>
- <execution>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <verbose>false</verbose>
- <failOnViolation>true</failOnViolation>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <failOnWarning>false</failOnWarning>
- <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
- <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
- <configLocation>${project.basedir}/../../../tools/maven/scalastyle-config.xml</configLocation>
- <outputFile>${project.basedir}/scalastyle-output.xml</outputFile>
- <outputEncoding>UTF-8</outputEncoding>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java b/flink-addons/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java
deleted file mode 100644
index 77d102d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.streaming.scala;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.typeutils.runtime.TupleSerializerBase;
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
-import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
-import org.apache.flink.streaming.api.function.aggregation.SumFunction;
-
-import scala.Product;
-
-public class ScalaStreamingAggregator<IN extends Product> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- TupleSerializerBase<IN> serializer;
- Object[] fields;
- int length;
- int position;
-
- public ScalaStreamingAggregator(TypeSerializer<IN> serializer, int pos) {
- this.serializer = (TupleSerializerBase<IN>) serializer;
- this.length = this.serializer.getArity();
- this.fields = new Object[this.length];
- this.position = pos;
- }
-
- public class Sum extends AggregationFunction<IN> {
- private static final long serialVersionUID = 1L;
- SumFunction sumFunction;
-
- public Sum(SumFunction func) {
- super(ScalaStreamingAggregator.this.position);
- this.sumFunction = func;
- }
-
- @Override
- public IN reduce(IN value1, IN value2) throws Exception {
- for (int i = 0; i < length; i++) {
- fields[i] = value2.productElement(i);
- }
-
- fields[position] = sumFunction.add(fields[position], value1.productElement(position));
-
- return serializer.createInstance(fields);
- }
- }
-
- public class ProductComparableAggregator extends ComparableAggregator<IN> {
-
- private static final long serialVersionUID = 1L;
-
- public ProductComparableAggregator(AggregationFunction.AggregationType aggregationType,
- boolean first) {
- super(ScalaStreamingAggregator.this.position, aggregationType, first);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public IN reduce(IN value1, IN value2) throws Exception {
- Object v1 = value1.productElement(position);
- Object v2 = value2.productElement(position);
-
- int c = comparator.isExtremal((Comparable<Object>) v1, v2);
-
- if (byAggregate) {
- if (c == 1) {
- return value1;
- }
- if (first) {
- if (c == 0) {
- return value1;
- }
- }
-
- return value2;
- } else {
- for (int i = 0; i < length; i++) {
- fields[i] = value2.productElement(i);
- }
-
- if (c == 1) {
- fields[position] = v1;
- }
-
- return serializer.createInstance(fields);
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
deleted file mode 100644
index d60e796..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
+++ /dev/null
@@ -1,380 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala
-
-import java.util
-
-import scala.collection.JavaConversions.asScalaBuffer
-import scala.reflect.ClassTag
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.streaming.api.scala._
-import org.apache.flink.streaming.api.datastream.{ConnectedDataStream => JavaCStream}
-import org.apache.flink.streaming.api.function.co.{ CoFlatMapFunction, CoMapFunction, CoReduceFunction, CoWindowFunction }
-import org.apache.flink.streaming.api.invokable.operator.co.{ CoFlatMapInvokable, CoMapInvokable, CoReduceInvokable }
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.util.Collector
-
-class ConnectedDataStream[IN1, IN2](javaStream: JavaCStream[IN1, IN2]) {
-
- /**
- * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
- * the output to a common type. The transformation calls a
- * @param fun1 for each element of the first input and
- * @param fun2 for each element of the second input. Each
- * CoMapFunction call returns exactly one element.
- *
- * The CoMapFunction used to jointly transform the two input
- * DataStreams
- * @return The transformed { @link DataStream}
- */
- def map[R: TypeInformation: ClassTag](fun1: IN1 => R, fun2: IN2 => R):
- DataStream[R] = {
- if (fun1 == null || fun2 == null) {
- throw new NullPointerException("Map function must not be null.")
- }
- val comapper = new CoMapFunction[IN1, IN2, R] {
- def map1(in1: IN1): R = clean(fun1)(in1)
- def map2(in2: IN2): R = clean(fun2)(in2)
- }
-
- new DataStream(javaStream.addCoFunction("map", implicitly[TypeInformation[R]],
- new CoMapInvokable[IN1, IN2, R](comapper)))
- }
-
- /**
- * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
- * the output to a common type. The transformation calls a
- * {@link CoMapFunction#map1} for each element of the first input and
- * {@link CoMapFunction#map2} for each element of the second input. Each
- * CoMapFunction call returns exactly one element. The user can also extend
- * {@link RichCoMapFunction} to gain access to other features provided by
- * the {@link RichFuntion} interface.
- *
- * @param coMapper
- * The CoMapFunction used to jointly transform the two input
- * DataStreams
- * @return The transformed { @link DataStream}
- */
- def map[R: TypeInformation: ClassTag](coMapper: CoMapFunction[IN1, IN2, R]):
- DataStream[R] = {
- if (coMapper == null) {
- throw new NullPointerException("Map function must not be null.")
- }
-
- new DataStream(javaStream.addCoFunction("map", implicitly[TypeInformation[R]],
- new CoMapInvokable[IN1, IN2, R](coMapper)))
- }
-
- /**
- * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
- * maps the output to a common type. The transformation calls a
- * {@link CoFlatMapFunction#flatMap1} for each element of the first input
- * and {@link CoFlatMapFunction#flatMap2} for each element of the second
- * input. Each CoFlatMapFunction call returns any number of elements
- * including none. The user can also extend {@link RichFlatMapFunction} to
- * gain access to other features provided by the {@link RichFuntion}
- * interface.
- *
- * @param coFlatMapper
- * The CoFlatMapFunction used to jointly transform the two input
- * DataStreams
- * @return The transformed { @link DataStream}
- */
- def flatMap[R: TypeInformation: ClassTag](coFlatMapper: CoFlatMapFunction[IN1, IN2, R]):
- DataStream[R] = {
- if (coFlatMapper == null) {
- throw new NullPointerException("FlatMap function must not be null.")
- }
- new DataStream[R](javaStream.addCoFunction("flatMap", implicitly[TypeInformation[R]],
- new CoFlatMapInvokable[IN1, IN2, R](coFlatMapper)))
- }
-
- /**
- * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
- * maps the output to a common type. The transformation calls a
- * @param fun1 for each element of the first input
- * and @param fun2 for each element of the second
- * input. Each CoFlatMapFunction call returns any number of elements
- * including none.
- *
- * @return The transformed { @link DataStream}
- */
- def flatMap[R: TypeInformation: ClassTag](fun1: (IN1, Collector[R]) => Unit,
- fun2: (IN2, Collector[R]) => Unit): DataStream[R] = {
- if (fun1 == null || fun2 == null) {
- throw new NullPointerException("FlatMap functions must not be null.")
- }
- val flatMapper = new CoFlatMapFunction[IN1, IN2, R] {
- def flatMap1(value: IN1, out: Collector[R]): Unit = clean(fun1)(value, out)
- def flatMap2(value: IN2, out: Collector[R]): Unit = clean(fun2)(value, out)
- }
- flatMap(flatMapper)
- }
-
- /**
- * GroupBy operation for connected data stream. Groups the elements of
- * input1 and input2 according to keyPosition1 and keyPosition2. Used for
- * applying function on grouped data streams for example
- * {@link ConnectedDataStream#reduce}
- *
- * @param keyPosition1
- * The field used to compute the hashcode of the elements in the
- * first input stream.
- * @param keyPosition2
- * The field used to compute the hashcode of the elements in the
- * second input stream.
- * @return @return The transformed { @link ConnectedDataStream}
- */
- def groupBy(keyPosition1: Int, keyPosition2: Int): ConnectedDataStream[IN1, IN2] = {
- javaStream.groupBy(keyPosition1, keyPosition2)
- }
-
- /**
- * GroupBy operation for connected data stream. Groups the elements of
- * input1 and input2 according to keyPositions1 and keyPositions2. Used for
- * applying function on grouped data streams for example
- * {@link ConnectedDataStream#reduce}
- *
- * @param keyPositions1
- * The fields used to group the first input stream.
- * @param keyPositions2
- * The fields used to group the second input stream.
- * @return @return The transformed { @link ConnectedDataStream}
- */
- def groupBy(keyPositions1: Array[Int], keyPositions2: Array[Int]):
- ConnectedDataStream[IN1, IN2] = {
- javaStream.groupBy(keyPositions1, keyPositions2)
- }
-
- /**
- * GroupBy operation for connected data stream using key expressions. Groups
- * the elements of input1 and input2 according to field1 and field2. A field
- * expression is either the name of a public field or a getter method with
- * parentheses of the {@link DataStream}S underlying type. A dot can be used
- * to drill down into objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field1
- * The grouping expression for the first input
- * @param field2
- * The grouping expression for the second input
- * @return The grouped { @link ConnectedDataStream}
- */
- def groupBy(field1: String, field2: String): ConnectedDataStream[IN1, IN2] = {
- javaStream.groupBy(field1, field2)
- }
-
- /**
- * GroupBy operation for connected data stream using key expressions. Groups
- * the elements of input1 and input2 according to fields1 and fields2. A
- * field expression is either the name of a public field or a getter method
- * with parentheses of the {@link DataStream}S underlying type. A dot can be
- * used to drill down into objects, as in {@code "field1.getInnerField2()" }
- * .
- *
- * @param fields1
- * The grouping expressions for the first input
- * @param fields2
- * The grouping expressions for the second input
- * @return The grouped { @link ConnectedDataStream}
- */
- def groupBy(fields1: Array[String], fields2: Array[String]):
- ConnectedDataStream[IN1, IN2] = {
- javaStream.groupBy(fields1, fields2)
- }
-
- /**
- * GroupBy operation for connected data stream. Groups the elements of
- * input1 and input2 using fun1 and fun2. Used for applying
- * function on grouped data streams for example
- * {@link ConnectedDataStream#reduce}
- *
- * @param fun1
- * The function used for grouping the first input
- * @param fun2
- * The function used for grouping the second input
- * @return @return The transformed { @link ConnectedDataStream}
- */
- def groupBy[K: TypeInformation](fun1: IN1 => _, fun2: IN2 => _):
- ConnectedDataStream[IN1, IN2] = {
-
- val keyExtractor1 = new KeySelector[IN1, Any] {
- def getKey(in: IN1) = clean(fun1)(in)
- }
- val keyExtractor2 = new KeySelector[IN2, Any] {
- def getKey(in: IN2) = clean(fun2)(in)
- }
-
- javaStream.groupBy(keyExtractor1, keyExtractor2)
- }
-
- /**
- * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
- * the outputs to a common type. If the {@link ConnectedDataStream} is
- * batched or windowed then the reduce transformation is applied on every
- * sliding batch/window of the data stream. If the connected data stream is
- * grouped then the reducer is applied on every group of elements sharing
- * the same key. This type of reduce is much faster than reduceGroup since
- * the reduce function can be applied incrementally.
- *
- * @param coReducer
- * The { @link CoReduceFunction} that will be called for every
- * element of the inputs.
- * @return The transformed { @link DataStream}.
- */
- def reduce[R: TypeInformation: ClassTag](coReducer: CoReduceFunction[IN1, IN2, R]):
- DataStream[R] = {
- if (coReducer == null) {
- throw new NullPointerException("Reduce function must not be null.")
- }
-
- new DataStream[R](javaStream.addCoFunction("coReduce", implicitly[TypeInformation[R]],
- new CoReduceInvokable[IN1, IN2, R](coReducer)))
- }
-
- /**
- * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
- * the outputs to a common type. If the {@link ConnectedDataStream} is
- * batched or windowed then the reduce transformation is applied on every
- * sliding batch/window of the data stream. If the connected data stream is
- * grouped then the reducer is applied on every group of elements sharing
- * the same key. This type of reduce is much faster than reduceGroup since
- * the reduce function can be applied incrementally.
- *
- * @param reducer1
- * @param reducer2
- * @param mapper1
- * @param mapper2
- *
- * @return The transformed { @link DataStream}.
- */
- def reduce[R: TypeInformation: ClassTag](reducer1: (IN1, IN1) => IN1,
- reducer2: (IN2, IN2) => IN2,mapper1: IN1 => R, mapper2: IN2 => R): DataStream[R] = {
- if (mapper1 == null || mapper2 == null) {
- throw new NullPointerException("Map functions must not be null.")
- }
- if (reducer1 == null || reducer2 == null) {
- throw new NullPointerException("Reduce functions must not be null.")
- }
-
- val reducer = new CoReduceFunction[IN1, IN2, R] {
- def reduce1(value1: IN1, value2: IN1): IN1 = clean(reducer1)(value1, value2)
- def map2(value: IN2): R = clean(mapper2)(value)
- def reduce2(value1: IN2, value2: IN2): IN2 = clean(reducer2)(value1, value2)
- def map1(value: IN1): R = clean(mapper1)(value)
- }
- reduce(reducer)
- }
-
- /**
- * Applies a CoWindow transformation on the connected DataStreams. The
- * transformation calls the {@link CoWindowFunction#coWindow} method for for
- * time aligned windows of the two data streams. System time is used as
- * default to compute windows.
- *
- * @param coWindowFunction
- * The { @link CoWindowFunction} that will be applied for the time
- * windows.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- *
- * @return The transformed { @link DataStream}.
- */
- def windowReduce[R: TypeInformation: ClassTag](coWindowFunction:
- CoWindowFunction[IN1, IN2, R], windowSize: Long, slideInterval: Long) = {
- if (coWindowFunction == null) {
- throw new NullPointerException("CoWindow function must no be null")
- }
-
- javaStream.windowReduce(coWindowFunction, windowSize, slideInterval)
- }
-
- /**
- * Applies a CoWindow transformation on the connected DataStreams. The
- * transformation calls the {@link CoWindowFunction#coWindow} method for for
- * time aligned windows of the two data streams. System time is used as
- * default to compute windows.
- *
- * @param coWindower
- * The coWindowing function to be applied for the time windows.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- *
- * @return The transformed { @link DataStream}.
- */
- def windowReduce[R: TypeInformation: ClassTag](coWindower: (Seq[IN1], Seq[IN2],
- Collector[R]) => Unit, windowSize: Long, slideInterval: Long) = {
- if (coWindower == null) {
- throw new NullPointerException("CoWindow function must no be null")
- }
-
- val coWindowFun = new CoWindowFunction[IN1, IN2, R] {
- def coWindow(first: util.List[IN1], second: util.List[IN2],
- out: Collector[R]): Unit = clean(coWindower)(first, second, out)
- }
-
- javaStream.windowReduce(coWindowFun, windowSize, slideInterval)
- }
-
- /**
- * Returns the first {@link DataStream}.
- *
- * @return The first DataStream.
- */
- def getFirst(): DataStream[IN1] = {
- javaStream.getFirst
- }
-
- /**
- * Returns the second {@link DataStream}.
- *
- * @return The second DataStream.
- */
- def getSecond(): DataStream[IN2] = {
- javaStream.getSecond
- }
-
- /**
- * Gets the type of the first input
- *
- * @return The type of the first input
- */
- def getInputType1(): TypeInformation[IN1] = {
- javaStream.getInputType1
- }
-
- /**
- * Gets the type of the second input
- *
- * @return The type of the second input
- */
- def getInputType2(): TypeInformation[IN2] = {
- javaStream.getInputType2
- }
-
-}
[10/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java
new file mode 100644
index 0000000..9d055d5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+
+/**
+ * This delta function calculates the euclidean distance between two given
+ * points.
+ *
+ * Euclidean distance: http://en.wikipedia.org/wiki/Euclidean_distance
+ *
+ * @param <DATA>
+ * The input data type. This delta function works with a double[],
+ * but can extract/convert to it from any other given object in case
+ * the respective extractor has been set. See
+ * {@link ExtractionAwareDeltaFunction} for more information.
+ */
+public class EuclideanDistance<DATA> extends ExtractionAwareDeltaFunction<DATA, double[]> {
+
+ public EuclideanDistance() {
+ super(null);
+ }
+
+ public EuclideanDistance(Extractor<DATA, double[]> converter) {
+ super(converter);
+ }
+
+ /**
+ * auto-generated version id
+ */
+ private static final long serialVersionUID = 3119432599634512359L;
+
+ @Override
+ public double getNestedDelta(double[] oldDataPoint, double[] newDataPoint) {
+ double result = 0;
+ for (int i = 0; i < oldDataPoint.length; i++) {
+ result += (oldDataPoint[i] - newDataPoint[i]) * (oldDataPoint[i] - newDataPoint[i]);
+ }
+ return Math.sqrt(result);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java
new file mode 100644
index 0000000..81d01a4
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+
+/**
+ * Extend this abstract class to implement a delta function which is aware of
+ * extracting the data on which the delta is calculated from a more complex data
+ * structure. For example in case you want to be able to run a delta only on one
+ * field of a Tuple type or only on some fields from an array.
+ *
+ * @param <DATA>
+ * The input data type. The input of this type will be passed to the
+ * extractor which will transform into a TO-object. The delta
+ * function then runs on this TO-object.
+ * @param <TO>
+ * The type on which the delta function runs. (The type of the delta
+ * function)
+ */
+public abstract class ExtractionAwareDeltaFunction<DATA, TO> implements DeltaFunction<DATA> {
+
+ /**
+ * Generated Version ID
+ */
+ private static final long serialVersionUID = 6927486219702689554L;
+ private Extractor<DATA, TO> converter;
+
+ public ExtractionAwareDeltaFunction(Extractor<DATA, TO> converter) {
+ this.converter = converter;
+ }
+
+ /**
+ * This method takes the two data point and runs the set extractor on it.
+ * The delta function implemented at {@link getNestedDelta} is then called
+ * with the extracted data. In case no extractor is set the input data gets
+ * passes to {@link getNestedDelta} as-is. The return value is just
+ * forwarded from {@link getNestedDelta}.
+ *
+ * @param oldDataPoint
+ * the older data point as raw data (before extraction).
+ * @param newDataPoint
+ * the new data point as raw data (before extraction).
+ * @return the delta between the two points.
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public double getDelta(DATA oldDataPoint, DATA newDataPoint) {
+ if (converter == null) {
+ // In case no conversion/extraction is required, we can cast DATA to
+ // TO
+ // => Therefore, "unchecked" warning is suppressed for this method.
+ return getNestedDelta((TO) oldDataPoint, (TO) newDataPoint);
+ } else {
+ return getNestedDelta(converter.extract(oldDataPoint), converter.extract(newDataPoint));
+ }
+
+ }
+
+ /**
+ * This method is exactly the same as
+ * {@link DeltaFunction#getDelta(Object, Object)} except that it gets the
+ * result of the previously done extractions as input. Therefore, this
+ * method only does the actual calculation of the delta but no data
+ * extraction or conversion.
+ *
+ * @param oldDataPoint
+ * the older data point.
+ * @param newDataPoint
+ * the new data point.
+ * @return the delta between the two points.
+ */
+ public abstract double getNestedDelta(TO oldDataPoint, TO newDataPoint);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java
new file mode 100644
index 0000000..ee878ac
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import org.apache.flink.api.java.tuple.Tuple;
+
+/**
+ * Converts a Tuple to an Object-Array. The field which should be included in
+ * the array can selected and reordered as needed.
+ */
+public class ArrayFromTuple implements Extractor<Tuple, Object[]> {
+
+ /**
+ * Auto generated version id
+ */
+ private static final long serialVersionUID = -6076121226427616818L;
+ int[] order = null;
+
+ /**
+ * Using this constructor the extractor will convert the whole tuple (all
+ * fields in the original order) to an array.
+ */
+ public ArrayFromTuple() {
+ // noting to do
+ }
+
+ /**
+ * Using this constructor the extractor will combine the fields as specified
+ * in the indexes parameter in an object array.
+ *
+ * @param indexes
+ * the field ids (enumerated from 0)
+ */
+ public ArrayFromTuple(int... indexes) {
+ this.order = indexes;
+ }
+
+ @Override
+ public Object[] extract(Tuple in) {
+ Object[] output;
+
+ if (order == null) {
+ // copy the whole tuple
+ output = new Object[in.getArity()];
+ for (int i = 0; i < in.getArity(); i++) {
+ output[i] = in.getField(i);
+ }
+ } else {
+ // copy user specified order
+ output = new Object[order.length];
+ for (int i = 0; i < order.length; i++) {
+ output[i] = in.getField(order[i]);
+ }
+ }
+
+ return output;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java
new file mode 100644
index 0000000..9048a3c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+/**
+ * Combines two extractors which will be executed one after each other.
+ *
+ * @param <FROM>
+ * The input type of the first extractor.
+ * @param <OVER>
+ * The output type of the first and the input type of the second
+ * extractor.
+ * @param <TO>
+ * The output type of the second extractor and the output type of the
+ * over all extraction.
+ */
+public class ConcatinatedExtract<FROM, OVER, TO> implements Extractor<FROM, TO> {
+
+ /**
+ * auto-generated id
+ */
+ private static final long serialVersionUID = -7807197760725651752L;
+
+ private Extractor<FROM, OVER> e1;
+ private Extractor<OVER, TO> e2;
+
+ /**
+ * Combines two extractors which will be executed one after each other.
+ *
+ * @param e1
+ * First extractor: This extractor gets applied to the input data
+ * first. Its output as then passed as input to the second
+ * extractor.
+ * @param e2
+ * Second extractor: This extractor gets the output of the first
+ * extractor as input. Its output is then the result of the over
+ * all extraction.
+ */
+ public ConcatinatedExtract(Extractor<FROM, OVER> e1, Extractor<OVER, TO> e2) {
+ this.e1 = e1;
+ this.e2 = e2;
+ }
+
+ @Override
+ public TO extract(FROM in) {
+ return e2.extract(e1.extract(in));
+ }
+
+ public <OUT> ConcatinatedExtract<FROM, TO, OUT> add(Extractor<TO, OUT> e3) {
+ return new ConcatinatedExtract<FROM, TO, OUT>(this, e3);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java
new file mode 100644
index 0000000..b103ca3
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import java.io.Serializable;
+
+/**
+ * Extractors allow to extract/convert one type to another. They are mostly used
+ * to extract some fields out of a more complex structure (Tuple/Array) to run
+ * further calculation on the extraction result.
+ *
+ * @param <FROM>
+ * The input data type.
+ * @param <TO>
+ * The output data type.
+ */
+public interface Extractor<FROM, TO> extends Serializable {
+
+ /**
+ * Extracts/Converts the given input to an object of the output type
+ *
+ * @param in
+ * the input data
+ * @return the extracted/converted data
+ */
+ public TO extract(FROM in);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java
new file mode 100644
index 0000000..0568276
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import java.lang.reflect.Array;
+
+/**
+ * Extracts a single field out of an array.
+ *
+ * @param <OUT>
+ * The type of the extracted field.
+ */
+public class FieldFromArray<OUT> implements Extractor<Object, OUT> {
+
+ /**
+ * Auto-gernated version id
+ */
+ private static final long serialVersionUID = -5161386546695574359L;
+ private int fieldId = 0;
+
+ /**
+ * Extracts the first field (id 0) from the array
+ */
+ public FieldFromArray() {
+ // noting to do => will use default 0
+ }
+
+ /**
+ * Extracts the field with the given id from the array.
+ *
+ * @param fieldId
+ * The id of the field which will be extracted from the array.
+ */
+ public FieldFromArray(int fieldId) {
+ this.fieldId = fieldId;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public OUT extract(Object in) {
+ return (OUT) Array.get(in, fieldId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java
new file mode 100644
index 0000000..07b38f5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import org.apache.flink.api.java.tuple.Tuple;
+
+/**
+ * Extracts a single field out of a tuple.
+ *
+ * @param <OUT>
+ * The type of the extracted field.
+ */
+public class FieldFromTuple<OUT> implements Extractor<Tuple, OUT> {
+
+ /**
+ * Auto-gernated version id
+ */
+ private static final long serialVersionUID = -5161386546695574359L;
+ private int fieldId = 0;
+
+ /**
+ * Extracts the first field (id 0) from the tuple
+ */
+ public FieldFromTuple() {
+ // noting to do => will use default 0
+ }
+
+ /**
+ * Extracts the field with the given id from the tuple.
+ *
+ * @param fieldId
+ * The id of the field which will be extracted from the tuple.
+ */
+ public FieldFromTuple(int fieldId) {
+ this.fieldId = fieldId;
+ }
+
+ @Override
+ public OUT extract(Tuple in) {
+ return in.getField(fieldId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java
new file mode 100644
index 0000000..4e98689
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import java.lang.reflect.Array;
+
+/**
+ * Extracts multiple fields from an array and puts them into a new array of the
+ * specified type.
+ *
+ * @param <OUT>
+ * The type of the output array. If out is set to String, the output
+ * of the extractor will be a String[]. If it is set to String[] the
+ * output will be String[][].
+ */
+public class FieldsFromArray<OUT> implements Extractor<Object, OUT[]> {
+
+ /**
+ * Auto-generated version id
+ */
+ private static final long serialVersionUID = 8075055384516397670L;
+ private int[] order;
+ private Class<OUT> clazz;
+
+ /**
+ * Extracts multiple fields from an array and puts them in the given order
+ * into a new array of the specified type.
+ *
+ * @param clazz
+ * the Class object representing the component type of the new
+ * array
+ * @param indexes
+ * The indexes of the fields to be extracted. Any order is
+ * possible, but not more than 255 fields due to limitations in
+ * {@link Array#newInstance(Class, int...)}.
+ */
+ public FieldsFromArray(Class<OUT> clazz, int... indexes) {
+ this.order = indexes;
+ this.clazz = clazz;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public OUT[] extract(Object in) {
+ OUT[] output = (OUT[]) Array.newInstance(clazz, order.length);
+ for (int i = 0; i < order.length; i++) {
+ output[i] = (OUT) Array.get(in, this.order[i]);
+ }
+ return output;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
new file mode 100644
index 0000000..1bfc461
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import org.apache.flink.api.java.tuple.Tuple;
+
+/**
+ * Extracts one or more fields of the type Double from a tuple and puts them
+ * into a new double[]
+ */
+public class FieldsFromTuple implements Extractor<Tuple, double[]> {
+
+ /**
+ * auto generated version id
+ */
+ private static final long serialVersionUID = -2554079091050273761L;
+ int[] indexes;
+
+ /**
+ * Extracts one or more fields of the the type Double from a tuple and puts
+ * them into a new double[] (in the specified order).
+ *
+ * @param indexes
+ * The indexes of the fields to be extracted.
+ */
+ public FieldsFromTuple(int... indexes) {
+ this.indexes = indexes;
+ }
+
+ @Override
+ public double[] extract(Tuple in) {
+ double[] out = new double[indexes.length];
+ for (int i = 0; i < indexes.length; i++) {
+ out[i] = (Double) in.getField(indexes[i]);
+ }
+ return out;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
new file mode 100644
index 0000000..840546f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+/**
+ * Represents a count based trigger or eviction policy. Use the
+ * {@link Count#of(int)} to get an instance.
+ */
+@SuppressWarnings("rawtypes")
+public class Count implements WindowingHelper {
+
+ private int count;
+ private int deleteOnEviction = 1;
+ private int startValue = CountTriggerPolicy.DEFAULT_START_VALUE;
+
+ /**
+ * Specifies on which element a trigger or an eviction should happen (based
+ * on the count of the elements).
+ *
+ * This constructor does exactly the same as {@link Count#of(int)}.
+ *
+ * @param count
+ * the number of elements to count before trigger/evict
+ */
+ public Count(int count) {
+ this.count = count;
+ }
+
+ @Override
+ public EvictionPolicy<?> toEvict() {
+ return new CountEvictionPolicy(count, deleteOnEviction);
+ }
+
+ @Override
+ public TriggerPolicy<?> toTrigger() {
+ return new CountTriggerPolicy(count, startValue);
+ }
+
+ /**
+ * Sets the number of elements deleted at each eviction (i.e when the number
+ * elements exceeds the window size). By default the elements get deleted
+ * one by one (deleteOnEvition = 1)
+ *
+ * @param deleteOnEviction
+ * The number of elements deleted at each evition
+ * @return Helper representing the count based policy
+ *
+ */
+ public Count withDelete(int deleteOnEviction) {
+ this.deleteOnEviction = deleteOnEviction;
+ return this;
+ }
+
+ /**
+ * Sets the initial value of the counter. 0 by default
+ *
+ * @param startValue
+ * Starting value of the window counter
+ * @return Helper representing the count based policy
+ *
+ */
+ public Count startingAt(int startValue) {
+ this.startValue = startValue;
+ return this;
+ }
+
+ /**
+ * Specifies a count based eviction (window size) or trigger policy (slide
+ * size). For eviction 'count' defines the number of elements in each
+ * window. For trigger 'count' defines how often do we call the user
+ * function in terms of number of elements received.
+ *
+ * @param count
+ * the number of elements to count before trigger/evict
+ * @return Helper representing the count based policy
+ */
+ public static Count of(int count) {
+ return new Count(count);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
new file mode 100644
index 0000000..5434a4e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
+import org.apache.flink.streaming.api.windowing.policy.DeltaPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+/**
+ * This helper represents a trigger or eviction policy based on a
+ * {@link DeltaFunction}.
+ *
+ * @param <DATA>
+ * the data type handled by the delta function represented by this
+ * helper.
+ */
+public class Delta<DATA> implements WindowingHelper<DATA> {
+
+ private DeltaFunction<DATA> deltaFunction;
+ private DATA initVal;
+ private double threshold;
+
+ /**
+ * Creates a delta helper representing a delta count or eviction policy
+ *
+ * @param deltaFunction
+ * The delta function which should be used to calculate the delta
+ * between points.
+ * @param initVal
+ * The initial value which will be used to calculate the first
+ * delta.
+ * @param threshold
+ * The threshold used by the delta function.
+ */
+ public Delta(DeltaFunction<DATA> deltaFunction, DATA initVal, double threshold) {
+ this.deltaFunction = deltaFunction;
+ this.initVal = initVal;
+ this.threshold = threshold;
+ }
+
+ @Override
+ public EvictionPolicy<DATA> toEvict() {
+ return new DeltaPolicy<DATA>(deltaFunction, initVal, threshold);
+ }
+
+ @Override
+ public TriggerPolicy<DATA> toTrigger() {
+ return new DeltaPolicy<DATA>(deltaFunction, initVal, threshold);
+ }
+
+ /**
+ * Creates a delta helper representing a delta trigger or eviction policy.
+ * </br></br> This policy calculates a delta between the data point which
+ * triggered last and the currently arrived data point. It triggers if the
+ * delta is higher than a specified threshold. </br></br> In case it gets
+ * used for eviction, this policy starts from the first element of the
+ * buffer and removes all elements from the buffer which have a higher delta
+ * then the threshold. As soon as there is an element with a lower delta,
+ * the eviction stops.
+ *
+ * @param deltaFunction
+ * The delta function which should be used to calculate the delta
+ * between points.
+ * @param initVal
+ * The initial value which will be used to calculate the first
+ * delta.
+ * @param threshold
+ * The threshold used by the delta function.
+ * @return Helper representing a delta trigger or eviction policy
+ */
+ public static <DATA> Delta<DATA> of(double threshold, DeltaFunction<DATA> deltaFunction,
+ DATA initVal) {
+ return new Delta<DATA>(deltaFunction, initVal, threshold);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java
new file mode 100644
index 0000000..8581ac5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+/**
+ * {@link Timestamp} implementation to be used when system time is needed to
+ * determine windows
+ */
+public class SystemTimestamp<T> implements Timestamp<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(T value) {
+ return System.currentTimeMillis();
+ }
+
+ public static <R> TimestampWrapper<R> getWrapper() {
+ return new TimestampWrapper<R>(new SystemTimestamp<R>(), System.currentTimeMillis());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
new file mode 100644
index 0000000..f94eea4
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+/**
+ * This helper represents a time based count or eviction policy. By default the
+ * time is measured with {@link System#currentTimeMillis()} in
+ * {@link DefaultTimeStamp}.
+ *
+ * @param <DATA>
+ * The data type which is handled by the time stamp used in the
+ * policy represented by this helper
+ */
+public class Time<DATA> implements WindowingHelper<DATA> {
+
+ protected long length;
+ protected TimeUnit granularity;
+ protected TimestampWrapper<DATA> timestampWrapper;
+ protected long delay;
+
+ /**
+ * Creates a helper representing a trigger which triggers every given
+ * length or an eviction which evicts all elements older than length.
+ *
+ * @param length
+ * The number of time units
+ * @param timeUnit
+ * The unit of time such as minute oder millisecond. Note that
+ * the smallest possible granularity is milliseconds. Any smaller
+ * time unit might cause an error at runtime due to conversion
+ * problems.
+ * @param timestamp
+ * The user defined timestamp that will be used to extract time
+ * information from the incoming elements
+ * @param startTime
+ * The startTime of the stream for computing the first window
+ */
+ private Time(long length, TimeUnit timeUnit, Timestamp<DATA> timestamp, long startTime) {
+ this(length, timeUnit, new TimestampWrapper<DATA>(timestamp, startTime));
+ }
+
+ /**
+ * Creates a helper representing a trigger which triggers every given
+ * length or an eviction which evicts all elements older than length.
+ *
+ * @param length
+ * The number of time units
+ * @param timeUnit
+ * The unit of time such as minute oder millisecond. Note that
+ * the smallest possible granularity is milliseconds. Any smaller
+ * time unit might cause an error at runtime due to conversion
+ * problems.
+ * @param timestampWrapper
+ * The user defined {@link TimestampWrapper} that will be used to
+ * extract time information from the incoming elements
+ */
+ private Time(long length, TimeUnit timeUnit, TimestampWrapper<DATA> timestampWrapper) {
+ this.length = length;
+ this.granularity = timeUnit;
+ this.timestampWrapper = timestampWrapper;
+ this.delay = 0;
+ }
+
+ @Override
+ public EvictionPolicy<DATA> toEvict() {
+ return new TimeEvictionPolicy<DATA>(granularityInMillis(), timestampWrapper);
+ }
+
+ @Override
+ public TriggerPolicy<DATA> toTrigger() {
+ return new TimeTriggerPolicy<DATA>(granularityInMillis(), timestampWrapper, delay);
+ }
+
+ /**
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using System time.
+ *
+ * @param length
+ * The number of time units
+ * @param timeUnit
+ * The unit of time such as minute oder millisecond. Note that
+ * the smallest possible granularity is milliseconds. Any smaller
+ * time unit might cause an error at runtime due to conversion
+ * problems.
+ * @return Helper representing the time based trigger and eviction policy
+ */
+ @SuppressWarnings("unchecked")
+ public static <DATA> Time<DATA> of(long length, TimeUnit timeUnit) {
+ return new Time<DATA>(length, timeUnit,
+ (TimestampWrapper<DATA>) SystemTimestamp.getWrapper());
+ }
+
+ /**
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using a user defined timestamp extractor.
+ *
+ * @param length
+ * The number of time units
+ * @param timestamp
+ * The user defined timestamp that will be used to extract time
+ * information from the incoming elements
+ * @param startTime
+ * The startTime used to compute the first window
+ * @return Helper representing the time based trigger and eviction policy
+ */
+ public static <DATA> Time<DATA> of(long length, Timestamp<DATA> timestamp, long startTime) {
+ return new Time<DATA>(length, null, timestamp, startTime);
+ }
+
+ /**
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using a user defined timestamp extractor. By
+ * default the start time is set to 0.
+ *
+ * @param length
+ * The number of time units
+ * @param timestamp
+ * The user defined timestamp that will be used to extract time
+ * information from the incoming elements
+ * @return Helper representing the time based trigger and eviction policy
+ */
+ public static <DATA> Time<DATA> of(long length, Timestamp<DATA> timestamp) {
+ return of(length, timestamp, 0);
+ }
+
+ /**
+ * Sets the delay for the first processed window.
+ *
+ * @param delay
+ * The number of time units before the first processed window.
+ * @return Helper representing the time based trigger and eviction policy
+ */
+ public Time<DATA> withDelay(long delay) {
+ this.delay = delay;
+ return this;
+ }
+
+ protected long granularityInMillis() {
+ return granularity == null ? length : granularity.toMillis(length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java
new file mode 100644
index 0000000..fea6020
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import java.io.Serializable;
+
+/**
+ * Interface for getting a timestamp from a custom value. Used in window
+ * reduces. In order to work properly, the timestamps must be non-decreasing.
+ *
+ * @param <T>
+ * Type of the value to create the timestamp from.
+ */
+public interface Timestamp<T> extends Serializable {
+
+ /**
+ * Values
+ *
+ * @param value
+ * The value to create the timestamp from
+ * @return The timestamp
+ */
+ public long getTimestamp(T value);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java
new file mode 100644
index 0000000..8c3a09d
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import java.io.Serializable;
+
+public class TimestampWrapper<T> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private long startTime;
+ private Timestamp<T> timestamp;
+
+ public TimestampWrapper(Timestamp<T> timeStamp, long startTime) {
+ this.timestamp = timeStamp;
+ this.startTime = startTime;
+ }
+
+ public long getTimestamp(T in) {
+ return timestamp.getTimestamp(in);
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public boolean isDefaultTimestamp() {
+ return timestamp instanceof SystemTimestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java
new file mode 100644
index 0000000..9df8432
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+/**
+ * A helper representing a count or eviction policy. Such helper classes are
+ * used to provide a nice and well readable API.
+ *
+ * @param <DATA>
+ * the type of input data handled by this helper
+ * @see Count
+ * @see Time
+ * @see Delta
+ */
+public interface WindowingHelper<DATA> {
+
+ public EvictionPolicy<DATA> toEvict();
+
+ public TriggerPolicy<DATA> toTrigger();
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java
new file mode 100644
index 0000000..29ba9eb
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * The {@link ActiveEvictionPolicy} wraps around a non active
+ * {@link EvictionPolicy}. It forwards all calls to
+ * {@link ActiveEvictionPolicy#notifyEvictionWithFakeElement(Object, int)} to
+ * {@link EvictionPolicy#notifyEviction(Object, boolean, int)} while the
+ * triggered parameter will be set to true.
+ *
+ * This class additionally implements the clone method and can wrap around
+ * {@link CloneableEvictionPolicy} to make it active.
+ *
+ * @param <DATA>
+ * The data type handled by this policy
+ */
+public class ActiveCloneableEvictionPolicyWrapper<DATA> extends ActiveEvictionPolicyWrapper<DATA>
+ implements CloneableEvictionPolicy<DATA> {
+
+ /**
+ * Auto generated version ID
+ */
+ private static final long serialVersionUID = 1520261575300622769L;
+ CloneableEvictionPolicy<DATA> nestedPolicy;
+
+ /**
+ * Creates a wrapper which activates the eviction policy which is wrapped
+ * in. This means that the nested policy will get called on fake elements as
+ * well as on real elements.
+ *
+ * This specialized version of the {@link ActiveEvictionPolicyWrapper} works
+ * with {@link CloneableEvictionPolicy} and is thereby cloneable as well.
+ *
+ * @param nestedPolicy
+ * The policy which should be activated/wrapped in.
+ */
+ public ActiveCloneableEvictionPolicyWrapper(CloneableEvictionPolicy<DATA> nestedPolicy) {
+ super(nestedPolicy);
+ this.nestedPolicy = nestedPolicy;
+ }
+
+ @Override
+ public ActiveCloneableEvictionPolicyWrapper<DATA> clone() {
+ return new ActiveCloneableEvictionPolicyWrapper<DATA>(nestedPolicy.clone());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
new file mode 100644
index 0000000..fe172bc
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * This interface is used for active eviction policies. beside the functionality
+ * inherited from {@link EvictionPolicy} it provides a method which gets called
+ * to notify on fake elements.
+ *
+ * In case an eviction policy implements this interface instead of the
+ * {@link EvictionPolicy} interface, not only the real but also fake data points
+ * will cause a notification of the eviction.
+ *
+ * Fake data points are mostly used in windowing based on time to trigger and
+ * evict even if no element arrives at all during a windows duration.
+ */
+public interface ActiveEvictionPolicy<DATA> extends EvictionPolicy<DATA> {
+
+ /**
+ * Proves if and how many elements should be deleted from the element
+ * buffer. The eviction takes place after the trigger and after the call to
+ * the UDF. This method is only called with fake elements.
+ *
+ * Note: Fake elements are always considered as triggered. Therefore this
+ * method does not have a triggered parameter.
+ *
+ * @param datapoint
+ * the current fake data point
+ * @param bufferSize
+ * the current size of the buffer (only real elements are
+ * counted)
+ * @return the number of elements to delete from the buffer (only real
+ * elements are counted)
+ */
+ public int notifyEvictionWithFakeElement(Object datapoint, int bufferSize);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
new file mode 100644
index 0000000..b3b6935
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * This {@link ActiveEvictionPolicy} wraps around a non active
+ * {@link EvictionPolicy}. It forwards all calls to
+ * {@link ActiveEvictionPolicy#notifyEvictionWithFakeElement(Object, int)} to
+ * {@link EvictionPolicy#notifyEviction(Object, boolean, int)} while the
+ * triggered parameter will be set to true.
+ *
+ * @param <DATA>
+ * The data type handled by this policy
+ */
+public class ActiveEvictionPolicyWrapper<DATA> implements ActiveEvictionPolicy<DATA> {
+
+ /**
+ * Auto generated version ID
+ */
+ private static final long serialVersionUID = -7656558669799505882L;
+ private EvictionPolicy<DATA> nestedPolicy;
+
+ /**
+ * Creates a wrapper which activates the eviction policy which is wrapped
+ * in. This means that the nested policy will get called on fake elements as
+ * well as on real elements.
+ *
+ * @param nestedPolicy
+ * The policy which should be activated/wrapped in.
+ */
+ public ActiveEvictionPolicyWrapper(EvictionPolicy<DATA> nestedPolicy) {
+ if (nestedPolicy == null) {
+ throw new RuntimeException("The nested policy must not be null.");
+ }
+ this.nestedPolicy = nestedPolicy;
+ }
+
+ @Override
+ public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
+ return nestedPolicy.notifyEviction(datapoint, triggered, bufferSize);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public int notifyEvictionWithFakeElement(Object datapoint, int bufferSize) {
+ return nestedPolicy.notifyEviction((DATA) datapoint, true, bufferSize);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
new file mode 100644
index 0000000..c44be37
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * In case an {@link ActiveTriggerPolicy} is used, it can implement own
+ * {@link Runnable} classes. Such {@link Runnable} classes will be executed as
+ * an own thread and can submit fake elements, to the element buffer at any
+ * time.
+ *
+ * The factory method for runnables of the {@link ActiveTriggerPolicy} gets an
+ * instance of this interface as parameter. The describes adding of elements can
+ * be done by the runnable using the methods provided in this interface.
+ *
+ */
+public interface ActiveTriggerCallback {
+
+ /**
+ * Submits a new fake data point to the element buffer. Such a fake element
+ * might be used to trigger at any time, but will never be included in the
+ * result of the reduce function. The submission of a fake element causes
+ * notifications only at the {@link ActiveTriggerPolicy} and
+ * {@link ActiveEvictionPolicy} implementations.
+ *
+ * @param datapoint
+ * the fake data point to be added
+ */
+ public void sendFakeElement(Object datapoint);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
new file mode 100644
index 0000000..abe5298
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+
+/**
+ * This interface extends the {@link TriggerPolicy} interface with functionality
+ * for active triggers. Active triggers can act in two ways:
+ *
+ * 1) Whenever an element arrives at the invokable, the
+ * {@link ActiveTriggerPolicy#preNotifyTrigger(Object)} method gets called
+ * first. It can return zero ore more fake data points which will be added
+ * before the currently arrived real element gets processed. This allows to
+ * handle empty windows in time based windowing with an user defined
+ * {@link Timestamp}. Triggers are not called on fake datapoint. A fake
+ * datapoint is always considered as triggered.
+ *
+ * 2) An active trigger has a factory method for a runnable. This factory method
+ * gets called at the start up of the invokable. The returned runnable will be
+ * executed in its own thread and can submit fake elements at any time threw an
+ * {@link ActiveTriggerCallback}. This allows to have time based triggers based
+ * on any system internal time measure. Triggers are not called on fake
+ * datapoint. A fake datapoints is always considered as triggered.
+ *
+ * @param <DATA>
+ * The data type which can be handled by this policy
+ */
+public interface ActiveTriggerPolicy<DATA> extends TriggerPolicy<DATA> {
+
+ /**
+ * Whenever an element arrives at the invokable, the
+ * {@link ActiveTriggerPolicy#preNotifyTrigger(Object)} method gets called
+ * first. It can return zero ore more fake data points which will be added
+ * before the the currently arrived real element gets processed. This allows
+ * to handle empty windows in time based windowing with an user defined
+ * {@link Timestamp}. Triggers are not called on fake datapoints. A fake
+ * datapoint is always considered as triggered.
+ *
+ * @param datapoint
+ * the data point which arrived at the invokable
+ * @return zero ore more fake data points which will be added before the the
+ * currently arrived real element gets processed.
+ */
+ public Object[] preNotifyTrigger(DATA datapoint);
+
+ /**
+ * This is the factory method for a runnable. This factory method gets
+ * called at the start up of the invokable. The returned runnable will be
+ * executed in its own thread and can submit fake elements at any time threw
+ * an {@link ActiveTriggerCallback}. This allows to have time based triggers
+ * based on any system internal time measure. Triggers are not called on
+ * fake datapoints. A fake datapoint is always considered as triggered.
+ *
+ * @param callback
+ * A callback object which allows to add fake elements from
+ * within the returned {@link Runnable}.
+ * @return The runnable implementation or null in case there is no. In case
+ * an {@link ActiveTriggerPolicy} is used, it can implement own
+ * {@link Runnable} classes. Such {@link Runnable} classes will be
+ * executed as an own thread and can submit fake elements, to the
+ * element buffer at any time.
+ */
+ public Runnable createActiveTriggerRunnable(ActiveTriggerCallback callback);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
new file mode 100644
index 0000000..132b495
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
+
+/**
+ * When used in {@link GroupedWindowInvokable}, eviction policies must
+ * provide a clone method. Eviction policies get cloned to provide an own
+ * instance for each group and respectively each individual element buffer as
+ * groups maintain their own buffers with the elements belonging to the
+ * respective group.
+ *
+ * This interface extends {@link EvictionPolicy} with such a clone method. It
+ * also adds the Java {@link Cloneable} interface as flag.
+ *
+ * @param <DATA>
+ * The data type handled by this policy
+ */
+public interface CloneableEvictionPolicy<DATA> extends EvictionPolicy<DATA>, Cloneable {
+
+ /**
+ * This method should return an exact copy of the object it belongs to
+ * including the current object state.
+ *
+ * @return a copy of this object
+ */
+ public CloneableEvictionPolicy<DATA> clone();
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
new file mode 100644
index 0000000..f5772a1
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
+
+/**
+ * When used in {@link GroupedWindowInvokable}, trigger policies can provide
+ * a clone method. Cloneable triggers can can be used in a distributed manner,
+ * which means they get cloned to provide an own instance for each group. This
+ * allows each group to trigger individually and only based on the elements
+ * belonging to the respective group.
+ *
+ * This interface extends {@link TriggerPolicy} with such a clone method. It
+ * also adds the Java {@link Cloneable} interface as flag.
+ *
+ * @param <DATA>
+ * The data type handled by this policy
+ */
+public interface CloneableTriggerPolicy<DATA> extends TriggerPolicy<DATA>, Cloneable {
+
+ /**
+ * This method should return an exact copy of the object it belongs to
+ * including the current object state.
+ *
+ * @return a copy of this object
+ */
+ public CloneableTriggerPolicy<DATA> clone();
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
new file mode 100644
index 0000000..a9e7b3f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * This eviction policy allows the eviction of data points from the buffer using
+ * a counter of arriving elements and a threshold (maximal buffer size)
+ *
+ * By default this policy does not react on fake elements. Wrap it in an
+ * {@link ActiveEvictionPolicyWrapper} to make it count even fake elements.
+ *
+ * @param <IN>
+ * the type of the incoming data points
+ */
+public class CountEvictionPolicy<IN> implements CloneableEvictionPolicy<IN> {
+
+ /**
+ * Auto generated version id
+ */
+ private static final long serialVersionUID = 2319201348806427996L;
+
+ int maxElements;
+ int counter;
+ int deleteOnEviction = 1;
+
+ /**
+ * This constructor allows the setup of the simplest possible count based
+ * eviction. It keeps the size of the buffer according to the given
+ * maxElements parameter by deleting the oldest element in the buffer.
+ * Eviction only takes place if the counter of arriving elements would be
+ * higher than maxElements without eviction.
+ *
+ * @param maxElements
+ * The maximum number of elements before eviction. As soon as one
+ * more element arrives, the oldest element will be deleted
+ */
+ public CountEvictionPolicy(int maxElements) {
+ this(maxElements, 1);
+ }
+
+ /**
+ * This constructor allows to set up both, the maximum number of elements
+ * and the number of elements to be deleted in case of an eviction.
+ *
+ * Eviction only takes place if the counter of arriving elements would be
+ * higher than maxElements without eviction. In such a case deleteOnEviction
+ * elements will be removed from the buffer.
+ *
+ * The counter of arriving elements is adjusted respectively, but never set
+ * below zero:
+ * counter=(counter-deleteOnEviction<0)?0:counter-deleteOnEviction
+ *
+ * @param maxElements
+ * maxElements The maximum number of elements before eviction.
+ * @param deleteOnEviction
+ * The number of elements to be deleted on eviction. The counter
+ * will be adjusted respectively but never below zero.
+ */
+ public CountEvictionPolicy(int maxElements, int deleteOnEviction) {
+ this(maxElements, deleteOnEviction, 0);
+ }
+
+ /**
+ * The same as {@link CountEvictionPolicy#CountEvictionPolicy(int, int)}.
+ * Additionally a custom start value for the counter of arriving elements
+ * can be set. By setting a negative start value the first eviction can be
+ * delayed.
+ *
+ * @param maxElements
+ * maxElements The maximum number of elements before eviction.
+ * @param deleteOnEviction
+ * The number of elements to be deleted on eviction. The counter
+ * will be adjusted respectively but never below zero.
+ * @param startValue
+ * A custom start value for the counter of arriving elements.
+ * @see CountEvictionPolicy#NextGenCountEvictionPolicy(int, int)
+ */
+ public CountEvictionPolicy(int maxElements, int deleteOnEviction, int startValue) {
+ this.counter = startValue;
+ this.deleteOnEviction = deleteOnEviction;
+ this.maxElements = maxElements;
+ }
+
+ @Override
+ public int notifyEviction(IN datapoint, boolean triggered, int bufferSize) {
+ // The comparison have to be >= and not == to cover case max=0
+ if (counter >= maxElements) {
+ // Adjust the counter according to the current eviction
+ counter = (counter - deleteOnEviction < 0) ? 0 : counter - deleteOnEviction;
+ // The current element will be added after the eviction
+ // Therefore, increase counter in any case
+ counter++;
+ return deleteOnEviction;
+ } else {
+ counter++;
+ return 0;
+ }
+ }
+
+ @Override
+ public CountEvictionPolicy<IN> clone() {
+ return new CountEvictionPolicy<IN>(maxElements, deleteOnEviction, counter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
new file mode 100644
index 0000000..7a3c75a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * This policy triggers at every n'th element.
+ *
+ * @param <IN>
+ * The type of the data points which are handled by this policy
+ */
+public class CountTriggerPolicy<IN> implements CloneableTriggerPolicy<IN> {
+
+ /**
+ * Auto generated version ID
+ */
+ private static final long serialVersionUID = -6357200688886103968L;
+
+ public static final int DEFAULT_START_VALUE = 0;
+
+ private int counter;
+ private int max;
+
+ /**
+ * This constructor will set up a count based trigger, which triggers after
+ * max elements have arrived.
+ *
+ * @param max
+ * The number of arriving elements before the trigger occurs.
+ */
+ public CountTriggerPolicy(int max) {
+ this(max, DEFAULT_START_VALUE);
+ }
+
+ /**
+ * In addition to {@link CountTriggerPolicy#CountTriggerPolicy(int)} this
+ * constructor allows to set a custom start value for the element counter.
+ * This can be used to delay the first trigger by setting a negative start
+ * value. Often the first trigger should be delayed in case of sliding
+ * windows. For example if the size of a window should be 4 and a trigger
+ * should happen every 2, a start value of -2 would allow to also have the
+ * first window of size 4.
+ *
+ * @param max
+ * The number of arriving elements before the trigger occurs.
+ * @param startValue
+ * The start value for the counter of arriving elements.
+ * @see CountTriggerPolicy#CountTriggerPolicy(int)
+ */
+ public CountTriggerPolicy(int max, int startValue) {
+ this.max = max;
+ this.counter = startValue;
+ }
+
+ @Override
+ public boolean notifyTrigger(IN datapoint) {
+ // The comparison have to be >= and not == to cover case max=0
+ if (counter >= max) {
+ // The current data point will be part of the next window!
+ // Therefore the counter needs to be set to one already.
+ counter = 1;
+ return true;
+ } else {
+ counter++;
+ return false;
+ }
+ }
+
+ @Override
+ public CountTriggerPolicy<IN> clone() {
+ return new CountTriggerPolicy<IN>(max, counter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
new file mode 100644
index 0000000..77bc692
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
+
+/**
+ * This policy calculates a delta between the data point which triggered last
+ * and the currently arrived data point. It triggers if the delta is higher than
+ * a specified threshold.
+ *
+ * In case it gets used for eviction, this policy starts from the first element
+ * of the buffer and removes all elements from the buffer which have a higher
+ * delta then the threshold. As soon as there is an element with a lower delta,
+ * the eviction stops.
+ *
+ * By default this policy does not react on fake elements. Wrap it in an
+ * {@link ActiveEvictionPolicyWrapper} to make it calculate the delta even on
+ * fake elements.
+ *
+ * @param <DATA>
+ * The type of the data points which are handled by this policy
+ */
+public class DeltaPolicy<DATA> implements CloneableTriggerPolicy<DATA>,
+ CloneableEvictionPolicy<DATA> {
+
+ /**
+ * Auto generated version ID
+ */
+ private static final long serialVersionUID = -7797538922123394967L;
+
+ private DeltaFunction<DATA> deltaFuntion;
+ private List<DATA> windowBuffer;
+ private double threshold;
+ private DATA triggerDataPoint;
+
+ /**
+ * Crates a delta policy which calculates a delta between the data point
+ * which triggered last and the currently arrived data point. It triggers if
+ * the delta is higher than a specified threshold.
+ *
+ * In case it gets used for eviction, this policy starts from the first
+ * element of the buffer and removes all elements from the buffer which have
+ * a higher delta then the threshold. As soon as there is an element with a
+ * lower delta, the eviction stops.
+ *
+ * @param deltaFuntion
+ * The delta function to be used.
+ * @param init
+ * The initial to be used for the calculation of a delta before
+ * the first trigger.
+ * @param threshold
+ * The threshold upon which a triggering should happen.
+ */
+ public DeltaPolicy(DeltaFunction<DATA> deltaFuntion, DATA init, double threshold) {
+ this.deltaFuntion = deltaFuntion;
+ this.triggerDataPoint = init;
+ this.windowBuffer = new LinkedList<DATA>();
+ this.threshold = threshold;
+ }
+
+ @Override
+ public boolean notifyTrigger(DATA datapoint) {
+ if (deltaFuntion.getDelta(this.triggerDataPoint, datapoint) > this.threshold) {
+ this.triggerDataPoint = datapoint;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
+ windowBuffer = windowBuffer.subList(windowBuffer.size() - bufferSize, bufferSize);
+ int evictCount = 0;
+ for (DATA bufferPoint : windowBuffer) {
+ if (deltaFuntion.getDelta(bufferPoint, datapoint) < this.threshold) {
+ break;
+ }
+ evictCount++;
+ }
+
+ if (evictCount > 0) {
+ windowBuffer = windowBuffer.subList(evictCount, windowBuffer.size());
+ }
+ windowBuffer.add(datapoint);
+ return evictCount;
+ }
+
+ @Override
+ public DeltaPolicy<DATA> clone() {
+ return new DeltaPolicy<DATA>(deltaFuntion, triggerDataPoint, threshold);
+ }
+}
[18/52] [partial] flink git commit: [FLINK-1452] Rename
'flink-addons' to 'flink-staging'
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
new file mode 100755
index 0000000..7ce864e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.rabbitmq;
+
+import java.io.IOException;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.connectors.ConnectorSource;
+import org.apache.flink.streaming.connectors.util.DeserializationSchema;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+
+public class RMQSource<OUT> extends ConnectorSource<OUT> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(RMQSource.class);
+
+ private final String QUEUE_NAME;
+ private final String HOST_NAME;
+
+ private transient ConnectionFactory factory;
+ private transient Connection connection;
+ private transient Channel channel;
+ private transient QueueingConsumer consumer;
+ private transient QueueingConsumer.Delivery delivery;
+
+ OUT out;
+
+ public RMQSource(String HOST_NAME, String QUEUE_NAME,
+ DeserializationSchema<OUT> deserializationSchema) {
+ super(deserializationSchema);
+ this.HOST_NAME = HOST_NAME;
+ this.QUEUE_NAME = QUEUE_NAME;
+ }
+
+ /**
+ * Initializes the connection to RMQ.
+ */
+ private void initializeConnection() {
+ factory = new ConnectionFactory();
+ factory.setHost(HOST_NAME);
+ try {
+ connection = factory.newConnection();
+ channel = connection.createChannel();
+ channel.queueDeclare(QUEUE_NAME, false, false, false, null);
+ consumer = new QueueingConsumer(channel);
+ channel.basicConsume(QUEUE_NAME, true, consumer);
+ } catch (IOException e) {
+ throw new RuntimeException("Cannot create RMQ connection with " + QUEUE_NAME + " at "
+ + HOST_NAME, e);
+ }
+ }
+
+ /**
+ * Called to forward the data from the source to the {@link DataStream}.
+ *
+ * @param collector
+ * The Collector for sending data to the dataStream
+ */
+ @Override
+ public void invoke(Collector<OUT> collector) throws Exception {
+
+ while (true) {
+
+ try {
+ delivery = consumer.nextDelivery();
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Cannot recieve RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
+ }
+ }
+
+ out = schema.deserialize(delivery.getBody());
+ if (schema.isEndOfStream(out)) {
+ break;
+ } else {
+ collector.collect(out);
+ }
+ }
+
+ }
+
+ @Override
+ public void open(Configuration config) {
+ initializeConnection();
+ }
+
+ @Override
+ public void close() {
+ try {
+ connection.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Error while closing RMQ connection with " + QUEUE_NAME
+ + " at " + HOST_NAME, e);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
new file mode 100755
index 0000000..a6ca9ae
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.rabbitmq;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.util.SerializationSchema;
+import org.apache.flink.streaming.connectors.util.SimpleStringSchema;
+
+public class RMQTopology {
+
+ public static void main(String[] args) throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+ @SuppressWarnings("unused")
+ DataStream<String> dataStream1 = env.addSource(
+ new RMQSource<String>("localhost", "hello", new SimpleStringSchema())).print();
+
+ @SuppressWarnings("unused")
+ DataStream<String> dataStream2 = env.fromElements("one", "two", "three", "four", "five",
+ "q").addSink(
+ new RMQSink<String>("localhost", "hello", new StringToByteSerializer()));
+
+ env.execute();
+ }
+
+ public static class StringToByteSerializer implements SerializationSchema<String, byte[]> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public byte[] serialize(String element) {
+ return element.getBytes();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
new file mode 100644
index 0000000..ddb2538
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.twitter;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.twitter.hbc.ClientBuilder;
+import com.twitter.hbc.core.Constants;
+import com.twitter.hbc.core.endpoint.StatusesSampleEndpoint;
+import com.twitter.hbc.core.processor.StringDelimitedProcessor;
+import com.twitter.hbc.httpclient.BasicClient;
+import com.twitter.hbc.httpclient.auth.Authentication;
+import com.twitter.hbc.httpclient.auth.OAuth1;
+
+/**
+ * Implementation of {@link SourceFunction} specialized to emit tweets from
+ * Twitter. It can connect to Twitter Streaming API, collect tweets and
+ */
+public class TwitterSource extends RichParallelSourceFunction<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TwitterSource.class);
+
+ private static final long serialVersionUID = 1L;
+ private String authPath;
+ private transient BlockingQueue<String> queue;
+ private int queueSize = 10000;
+ private transient BasicClient client;
+ private int waitSec = 5;
+
+ private boolean streaming;
+ private int numberOfTweets;
+
+ /**
+ * Create {@link TwitterSource} for streaming
+ *
+ * @param authPath
+ * Location of the properties file containing the required
+ * authentication information.
+ */
+ public TwitterSource(String authPath) {
+ this.authPath = authPath;
+ streaming = true;
+ }
+
+ /**
+ * Create {@link TwitterSource} to collect finite number of tweets
+ *
+ * @param authPath
+ * Location of the properties file containing the required
+ * authentication information.
+ * @param numberOfTweets
+ *
+ */
+ public TwitterSource(String authPath, int numberOfTweets) {
+ this.authPath = authPath;
+ streaming = false;
+ this.numberOfTweets = numberOfTweets;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ initializeConnection();
+ }
+
+ @Override
+ public void invoke(Collector<String> collector) throws Exception {
+
+ if (streaming) {
+ collectMessages(collector);
+ } else {
+ collectFiniteMessages(collector);
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ closeConnection();
+ }
+
+ /**
+ * Initialize Hosebird Client to be able to consume Twitter's Streaming API
+ */
+ private void initializeConnection() {
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Initializing Twitter Streaming API connection");
+ }
+
+ queue = new LinkedBlockingQueue<String>(queueSize);
+
+ StatusesSampleEndpoint endpoint = new StatusesSampleEndpoint();
+ endpoint.stallWarnings(false);
+
+ Authentication auth = authenticate();
+
+ initializeClient(endpoint, auth);
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Twitter Streaming API connection established successfully");
+ }
+ }
+
+ private OAuth1 authenticate() {
+
+ Properties authenticationProperties = loadAuthenticationProperties();
+
+ return new OAuth1(authenticationProperties.getProperty("consumerKey"),
+ authenticationProperties.getProperty("consumerSecret"),
+ authenticationProperties.getProperty("token"),
+ authenticationProperties.getProperty("secret"));
+ }
+
+ /**
+ * Reads the given properties file for the authentication data.
+ *
+ * @return the authentication data.
+ */
+ private Properties loadAuthenticationProperties() {
+ Properties properties = new Properties();
+ try {
+ InputStream input = new FileInputStream(authPath);
+ properties.load(input);
+ input.close();
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot open .properties file: " + authPath, e);
+ }
+ return properties;
+ }
+
+ private void initializeClient(StatusesSampleEndpoint endpoint, Authentication auth) {
+
+ client = new ClientBuilder().name("twitterSourceClient").hosts(Constants.STREAM_HOST)
+ .endpoint(endpoint).authentication(auth)
+ .processor(new StringDelimitedProcessor(queue)).build();
+
+ client.connect();
+ }
+
+ /**
+ * Put tweets into collector
+ *
+ * @param collector
+ * Collector in which the tweets are collected.
+ */
+ protected void collectFiniteMessages(Collector<String> collector) {
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Collecting tweets");
+ }
+
+ for (int i = 0; i < numberOfTweets; i++) {
+ collectOneMessage(collector);
+ }
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Collecting tweets finished");
+ }
+ }
+
+ /**
+ * Put tweets into collector
+ *
+ * @param collector
+ * Collector in which the tweets are collected.
+ */
+ protected void collectMessages(Collector<String> collector) {
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Tweet-stream begins");
+ }
+
+ while (true) {
+ collectOneMessage(collector);
+ }
+ }
+
+ /**
+ * Put one tweet into the collector.
+ *
+ * @param collector
+ * Collector in which the tweets are collected.
+ */
+ protected void collectOneMessage(Collector<String> collector) {
+ if (client.isDone()) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Client connection closed unexpectedly: {}", client.getExitEvent()
+ .getMessage());
+ }
+ }
+
+ try {
+ String msg = queue.poll(waitSec, TimeUnit.SECONDS);
+ if (msg != null) {
+ collector.collect(msg);
+ } else {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Did not receive a message in {} seconds", waitSec);
+ }
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException("'Waiting for tweet' thread is interrupted", e);
+ }
+
+ }
+
+ private void closeConnection() {
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Initiating connection close");
+ }
+
+ client.stop();
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Connection closed successfully");
+ }
+ }
+
+ /**
+ * Get the size of the queue in which the tweets are contained temporarily.
+ *
+ * @return the size of the queue in which the tweets are contained temporarily
+ */
+ public int getQueueSize() {
+ return queueSize;
+ }
+
+ /**
+ * Set the size of the queue in which the tweets are contained temporarily.
+ *
+ * @param queueSize
+ * The desired value.
+ */
+ public void setQueueSize(int queueSize) {
+ this.queueSize = queueSize;
+ }
+
+ /**
+ * This function tells how long TwitterSource waits for the tweets.
+ *
+ * @return Number of second.
+ */
+ public int getWaitSec() {
+ return waitSec;
+ }
+
+ /**
+ * This function sets how long TwitterSource should wait for the tweets.
+ *
+ * @param waitSec
+ * The desired value.
+ */
+ public void setWaitSec(int waitSec) {
+ this.waitSec = waitSec;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
new file mode 100644
index 0000000..a32fe1b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.twitter;
+
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
+import org.apache.flink.util.Collector;
+import org.apache.sling.commons.json.JSONException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TwitterStreaming {
+
+ private static final int PARALLELISM = 1;
+ private static final int SOURCE_PARALLELISM = 1;
+ private static final int NUMBEROFTWEETS = 100;
+
+ private static final Logger LOG = LoggerFactory.getLogger(TwitterStreaming.class);
+
+ public static class TwitterSink implements SinkFunction<Tuple5<Long, Integer, String, String, String>> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void invoke(Tuple5<Long, Integer, String, String, String> tuple) {
+ System.out.println("ID: " + tuple.f0 + " int: " + tuple.f1 + " LANGUAGE: " + tuple.f2);
+ System.out.println("NAME: " + tuple.f4);
+ System.out.println("TEXT: " + tuple.f3);
+ System.out.println("");
+ }
+
+ }
+
+ public static class SelectDataFlatMap extends
+ JSONParseFlatMap<String, Tuple5<Long, Integer, String, String, String>> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(String value, Collector<Tuple5<Long, Integer, String, String, String>> out)
+ throws Exception {
+ try {
+ out.collect(new Tuple5<Long, Integer, String, String, String>(
+ getLong(value, "id"),
+ getInt(value, "entities.hashtags[0].indices[1]"),
+ getString(value, "lang"),
+ getString(value, "text"),
+ getString(value, "user.name")));
+ } catch (JSONException e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Field not found");
+ }
+ }
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ String path = new String();
+
+ if (args != null && args.length == 1) {
+ path = args[0];
+ } else {
+ System.err.println("USAGE:\nTwitterStreaming <pathToPropertiesFile>");
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment
+ .createLocalEnvironment(PARALLELISM);
+
+ DataStream<String> streamSource = env.addSource(new TwitterSource(path, NUMBEROFTWEETS))
+ .setParallelism(SOURCE_PARALLELISM);
+
+ DataStream<Tuple5<Long, Integer, String, String, String>> selectedDataStream = streamSource
+ .flatMap(new SelectDataFlatMap());
+
+ selectedDataStream.addSink(new TwitterSink());
+
+ env.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
new file mode 100644
index 0000000..4bc6df0
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.twitter;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
+import org.apache.flink.util.Collector;
+import org.apache.sling.commons.json.JSONException;
+
+/**
+ * This program demonstrate the use of TwitterSource.
+ * Its aim is to count the frequency of the languages of tweets
+ */
+public class TwitterTopology {
+
+ private static final int NUMBEROFTWEETS = 100;
+
+ /**
+ * FlatMapFunction to determine the language of tweets if possible
+ */
+ public static class SelectLanguageFlatMap extends
+ JSONParseFlatMap<String, String> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Select the language from the incoming JSON text
+ */
+ @Override
+ public void flatMap(String value, Collector<String> out) throws Exception {
+ try{
+ out.collect(getString(value, "lang"));
+ }
+ catch (JSONException e){
+ out.collect("");
+ }
+ }
+
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ String path = new String();
+
+ if (args != null && args.length == 1) {
+ path = args[0];
+ } else {
+ System.err.println("USAGE:\nTwitterLocal <pathToPropertiesFile>");
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<String> streamSource = env.addSource(new TwitterSource(path, NUMBEROFTWEETS));
+
+
+ DataStream<Tuple2<String, Integer>> dataStream = streamSource
+ .flatMap(new SelectLanguageFlatMap())
+ .map(new MapFunction<String, Tuple2<String, Integer>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<String, Integer> map(String value) throws Exception {
+ return new Tuple2<String, Integer>(value, 1);
+ }
+ })
+ .groupBy(0)
+ .sum(1);
+
+ dataStream.print();
+
+ env.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java
new file mode 100644
index 0000000..4507a1d
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+import java.io.Serializable;
+
+public interface DeserializationSchema<T> extends Serializable {
+
+ /**
+ * Deserializes the incoming data.
+ *
+ * @param message
+ * The incoming message in a byte array
+ * @return The deserialized message in the required format.
+ */
+ public T deserialize(byte[] message);
+
+ /**
+ * Method to decide whether the element signals the end of the stream. If
+ * true is returned the element won't be emitted
+ *
+ * @param nextElement
+ * The element to test for end signal
+ * @return The end signal, if true the stream shuts down
+ */
+ public boolean isEndOfStream(T nextElement);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java
new file mode 100644
index 0000000..29c749a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+public class RawSchema implements DeserializationSchema<byte[]>,
+ SerializationSchema<byte[], byte[]> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public byte[] deserialize(byte[] message) {
+ return message;
+ }
+
+ @Override
+ public boolean isEndOfStream(byte[] nextElement) {
+ return false;
+ }
+
+ @Override
+ public byte[] serialize(byte[] element) {
+ return element;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java
new file mode 100644
index 0000000..7c32312
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+import java.io.Serializable;
+
+public interface SerializationSchema<T,R> extends Serializable {
+
+ /**
+ * Serializes the incoming element to a specified type.
+ *
+ * @param element
+ * The incoming element to be serialized
+ * @return The serialized element.
+ */
+ public R serialize(T element);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java
new file mode 100644
index 0000000..4b21580
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+public class SimpleStringSchema implements DeserializationSchema<String>,
+ SerializationSchema<String, String> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String deserialize(byte[] message) {
+ return new String(message);
+ }
+
+ @Override
+ public boolean isEndOfStream(String nextElement) {
+ return false;
+ }
+
+ @Override
+ public String serialize(String element) {
+ return element;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/db/DBStateTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/db/DBStateTest.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/db/DBStateTest.java
new file mode 100644
index 0000000..fe3783b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/db/DBStateTest.java
@@ -0,0 +1,105 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.flink.streaming.connectors.db;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.HashMap;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class DBStateTest {
+
+ public void stateTest(DBState<String, Integer> state) {
+ state.put("k1", 1);
+ assertEquals(Integer.valueOf(1), state.get("k1"));
+
+ state.put("k2", 2);
+ state.put("k3", 3);
+ assertEquals(Integer.valueOf(2), state.get("k2"));
+ assertEquals(Integer.valueOf(3), state.get("k3"));
+ state.remove("k2");
+
+ try {
+ state.get("k2");
+ fail();
+ } catch (Exception e) {
+ }
+ }
+
+ private void iteratorTest(DBStateWithIterator<String, Integer> state) {
+ HashMap<String, Integer> expected = new HashMap<String, Integer>();
+ HashMap<String, Integer> result = new HashMap<String, Integer>();
+
+ state.put("10", 10);
+ state.put("20", 20);
+ state.put("30", 30);
+
+ expected.put("10", 10);
+ expected.put("20", 20);
+ expected.put("30", 30);
+
+ DBStateIterator<String, Integer> iterator = state.getIterator();
+ while (iterator.hasNext()) {
+ String key = iterator.getNextKey();
+ Integer value = iterator.getNextValue();
+ result.put(key, value);
+ iterator.next();
+ }
+ state.close();
+
+ assertEquals(expected, result);
+ }
+
+ // TODO
+ @Ignore("Creates files with no licenses")
+ @Test
+ public void levelDBTest() {
+ LevelDBState<String, Integer> state = new LevelDBState<String, Integer>("test");
+ stateTest(state);
+ state.close();
+
+ state = new LevelDBState<String, Integer>("test");
+ iteratorTest(state);
+ state.close();
+ }
+
+ @Ignore("Needs running Memcached")
+ @Test
+ public void memcachedTest() {
+ MemcachedState<Integer> state = new MemcachedState<Integer>();
+ stateTest(state);
+ state.close();
+ }
+
+ @Ignore("Needs running Redis")
+ @Test
+ public void redisTest() {
+ RedisState<String, Integer> state = new RedisState<String, Integer>();
+ stateTest(state);
+ state.close();
+
+ state = new RedisState<String, Integer>();
+ iteratorTest(state);
+ state.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
new file mode 100644
index 0000000..b1d4115
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.json;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.flink.streaming.connectors.json.JSONParser;
+import org.apache.sling.commons.json.JSONException;
+import org.apache.sling.commons.json.JSONObject;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class JSONParserTest {
+
+ private String jsonText;
+ private String searchedField;
+
+ public JSONParserTest(String text, String field) {
+ jsonText = text;
+ searchedField = field;
+ }
+
+ @Parameters
+ public static Collection<Object[]> initParameterList() {
+
+ Object[][] parameterList = new Object[][] {
+ { "{\"key\":\"value\"}", "key" },
+ { "{\"key\":[\"value\"]}", "key[0]" },
+ { "{\"key\":[{\"key\":\"value\"}]}", "key[0].key" },
+ { "{\"key\":[{\"key\":[{\"key\":\"value\"}]}]}", "key[0].key[0].key"},
+ { "{\"key\":[1,[{\"key\":\"value\"}]]}", "key[1][0].key" },
+ { "{\"key\":[1,[[\"key\",2,\"value\"]]]}", "key[1][0][2]" },
+ { "{\"key\":{\"key\":{\"otherKey\":\"wrongValue\",\"key\":\"value\"},\"otherKey\":\"wrongValue\"},\"otherKey\":\"wrongValue\"}" , "key.key.key"}
+ };
+
+ return Arrays.asList(parameterList);
+ }
+
+ @Test
+ public void test() {
+ try {
+ JSONParser parser = new JSONParser(jsonText);
+ JSONObject jo = parser.parse(searchedField);
+ String expected = "{\"retValue\":\"value\"}";
+
+ assertTrue(expected.equals(jo.toString()));
+ }
+ catch (JSONException e) {
+ fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
new file mode 100644
index 0000000..8851086
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.json;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.flink.streaming.connectors.json.JSONParser;
+import org.apache.sling.commons.json.JSONException;
+import org.apache.sling.commons.json.JSONObject;
+import org.junit.Test;
+
+
+public class JSONParserTest2 {
+
+ @Test
+ public void testGetBooleanFunction() {
+ String jsonText = "{\"key\":true}";
+ String searchedField = "key";
+ try {
+ JSONParser parser = new JSONParser(jsonText);
+ JSONObject jo = parser.parse(searchedField);
+
+ assertTrue(jo.getBoolean("retValue"));
+ }
+ catch (JSONException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testGetDoubleFunction() {
+ double expected = 12345.12345;
+ String jsonText = "{\"key\":" + expected + "}";
+ String searchedField = "key";
+ try {
+ JSONParser parser = new JSONParser(jsonText);
+ JSONObject jo = parser.parse(searchedField);
+
+ assertEquals(expected,jo.getDouble("retValue"),0.000001);
+ }
+ catch (JSONException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testGetIntFunction() {
+ int expected = 15;
+ String jsonText = "{\"key\":" + expected + "}";
+ String searchedField = "key";
+ try {
+ JSONParser parser = new JSONParser(jsonText);
+ JSONObject jo = parser.parse(searchedField);
+
+ assertEquals(expected,jo.getInt("retValue"));
+ }
+ catch (JSONException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testGetLongFunction() {
+ long expected = 111111111111L;
+ String jsonText = "{\"key\":" + expected + "}";
+ String searchedField = "key";
+ try {
+ JSONParser parser = new JSONParser(jsonText);
+ JSONObject jo = parser.parse(searchedField);
+
+ assertEquals(expected,jo.getLong("retValue"));
+ }
+ catch (JSONException e) {
+ fail();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..2fb9345
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
@@ -0,0 +1,19 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..45b3b92
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml
@@ -0,0 +1,30 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+ <logger name="org.apache.flink.streaming" level="WARN"/>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/pom.xml b/flink-staging/flink-streaming/flink-streaming-core/pom.xml
new file mode 100644
index 0000000..a60bd01
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/pom.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-streaming-core</artifactId>
+ <name>flink-streaming-core</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math</artifactId>
+ <version>2.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.sling</groupId>
+ <artifactId>org.apache.sling.commons.json</artifactId>
+ <version>2.0.6</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
new file mode 100644
index 0000000..1d51216
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.SerializationException;
+import org.apache.commons.lang3.SerializationUtils;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.api.streamvertex.StreamVertexException;
+import org.apache.flink.streaming.partitioner.ShufflePartitioner;
+import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.streaming.state.OperatorState;
+import org.apache.flink.util.InstantiationUtil;
+
+public class StreamConfig implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final String INPUT_TYPE = "inputType_";
+ private static final String NUMBER_OF_OUTPUTS = "numberOfOutputs";
+ private static final String NUMBER_OF_INPUTS = "numberOfInputs";
+ private static final String CHAINED_OUTPUTS = "chainedOutputs";
+ private static final String CHAINED_TASK_CONFIG = "chainedTaskConfig_";
+ private static final String IS_CHAINED_VERTEX = "isChainedSubtask";
+ private static final String OUTPUT_NAME = "outputName_";
+ private static final String PARTITIONER_OBJECT = "partitionerObject_";
+ private static final String VERTEX_NAME = "vertexName";
+ private static final String ITERATION_ID = "iteration-id";
+ private static final String OUTPUT_SELECTOR = "outputSelector";
+ private static final String DIRECTED_EMIT = "directedEmit";
+ private static final String SERIALIZEDUDF = "serializedudf";
+ private static final String USER_FUNCTION = "userfunction";
+ private static final String BUFFER_TIMEOUT = "bufferTimeout";
+ private static final String OPERATOR_STATES = "operatorStates";
+ private static final String TYPE_SERIALIZER_IN_1 = "typeSerializer_in_1";
+ private static final String TYPE_SERIALIZER_IN_2 = "typeSerializer_in_2";
+ private static final String TYPE_SERIALIZER_OUT_1 = "typeSerializer_out_1";
+ private static final String TYPE_SERIALIZER_OUT_2 = "typeSerializer_out_2";
+ private static final String ITERATON_WAIT = "iterationWait";
+ private static final String OUTPUTS = "outVertexNames";
+ private static final String EDGES_IN_ORDER = "rwOrder";
+
+ // DEFAULT VALUES
+
+ private static final long DEFAULT_TIMEOUT = 100;
+
+ // CONFIG METHODS
+
+ private Configuration config;
+
+ public StreamConfig(Configuration config) {
+ this.config = config;
+ }
+
+ public Configuration getConfiguration() {
+ return config;
+ }
+
+ public void setVertexName(String vertexName) {
+ config.setString(VERTEX_NAME, vertexName);
+ }
+
+ public String getTaskName() {
+ return config.getString(VERTEX_NAME, "Missing");
+ }
+
+ public void setTypeSerializerIn1(StreamRecordSerializer<?> serializer) {
+ setTypeSerializer(TYPE_SERIALIZER_IN_1, serializer);
+ }
+
+ public void setTypeSerializerIn2(StreamRecordSerializer<?> serializer) {
+ setTypeSerializer(TYPE_SERIALIZER_IN_2, serializer);
+ }
+
+ public void setTypeSerializerOut1(StreamRecordSerializer<?> serializer) {
+ setTypeSerializer(TYPE_SERIALIZER_OUT_1, serializer);
+ }
+
+ public void setTypeSerializerOut2(StreamRecordSerializer<?> serializer) {
+ setTypeSerializer(TYPE_SERIALIZER_OUT_2, serializer);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> StreamRecordSerializer<T> getTypeSerializerIn1(ClassLoader cl) {
+ try {
+ return (StreamRecordSerializer<T>) InstantiationUtil.readObjectFromConfig(this.config,
+ TYPE_SERIALIZER_IN_1, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate serializer.");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> StreamRecordSerializer<T> getTypeSerializerIn2(ClassLoader cl) {
+ try {
+ return (StreamRecordSerializer<T>) InstantiationUtil.readObjectFromConfig(this.config,
+ TYPE_SERIALIZER_IN_2, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate serializer.");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> StreamRecordSerializer<T> getTypeSerializerOut1(ClassLoader cl) {
+ try {
+ return (StreamRecordSerializer<T>) InstantiationUtil.readObjectFromConfig(this.config,
+ TYPE_SERIALIZER_OUT_1, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate serializer.");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> StreamRecordSerializer<T> getTypeSerializerOut2(ClassLoader cl) {
+ try {
+ return (StreamRecordSerializer<T>) InstantiationUtil.readObjectFromConfig(this.config,
+ TYPE_SERIALIZER_OUT_2, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate serializer.");
+ }
+ }
+
+ private void setTypeSerializer(String key, StreamRecordSerializer<?> typeWrapper) {
+ config.setBytes(key, SerializationUtils.serialize(typeWrapper));
+ }
+
+ public void setBufferTimeout(long timeout) {
+ config.setLong(BUFFER_TIMEOUT, timeout);
+ }
+
+ public long getBufferTimeout() {
+ return config.getLong(BUFFER_TIMEOUT, DEFAULT_TIMEOUT);
+ }
+
+ public void setUserInvokable(StreamInvokable<?, ?> invokableObject) {
+ if (invokableObject != null) {
+ config.setClass(USER_FUNCTION, invokableObject.getClass());
+
+ try {
+ config.setBytes(SERIALIZEDUDF, SerializationUtils.serialize(invokableObject));
+ } catch (SerializationException e) {
+ throw new RuntimeException("Cannot serialize invokable object "
+ + invokableObject.getClass(), e);
+ }
+ }
+ }
+
+ @SuppressWarnings({ "unchecked" })
+ public <T> T getUserInvokable(ClassLoader cl) {
+ try {
+ return (T) InstantiationUtil.readObjectFromConfig(this.config, SERIALIZEDUDF, cl);
+ } catch (Exception e) {
+ throw new StreamVertexException("Cannot instantiate user function", e);
+ }
+ }
+
+ public void setDirectedEmit(boolean directedEmit) {
+ config.setBoolean(DIRECTED_EMIT, directedEmit);
+ }
+
+ public boolean isDirectedEmit() {
+ return config.getBoolean(DIRECTED_EMIT, false);
+ }
+
+
+ public void setOutputSelectors(List<OutputSelector<?>> outputSelector) {
+ try {
+ if (outputSelector != null) {
+ setDirectedEmit(true);
+ config.setBytes(OUTPUT_SELECTOR, SerializationUtils.serialize((Serializable) outputSelector));
+ }
+ } catch (SerializationException e) {
+ throw new RuntimeException("Cannot serialize OutputSelector");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> List<OutputSelector<T>> getOutputSelectors(ClassLoader cl) {
+ try {
+ return (List<OutputSelector<T>>) InstantiationUtil.readObjectFromConfig(this.config,
+ OUTPUT_SELECTOR, cl);
+ } catch (Exception e) {
+ throw new StreamVertexException("Cannot deserialize and instantiate OutputSelector", e);
+ }
+ }
+
+ public void setIterationId(Integer iterationId) {
+ config.setInteger(ITERATION_ID, iterationId);
+ }
+
+ public Integer getIterationId() {
+ return config.getInteger(ITERATION_ID, 0);
+ }
+
+ public void setIterationWaitTime(long time) {
+ config.setLong(ITERATON_WAIT, time);
+ }
+
+ public long getIterationWaitTime() {
+ return config.getLong(ITERATON_WAIT, 0);
+ }
+
+ public <T> void setPartitioner(String output, StreamPartitioner<T> partitionerObject) {
+
+ config.setBytes(PARTITIONER_OBJECT + output,
+ SerializationUtils.serialize(partitionerObject));
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> StreamPartitioner<T> getPartitioner(ClassLoader cl, String output) {
+ StreamPartitioner<T> partitioner = null;
+ try {
+ partitioner = (StreamPartitioner<T>) InstantiationUtil.readObjectFromConfig(
+ this.config, PARTITIONER_OBJECT + output, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Partitioner could not be instantiated.");
+ }
+ if (partitioner != null) {
+ return partitioner;
+ } else {
+ return new ShufflePartitioner<T>();
+ }
+ }
+
+ public void setSelectedNames(String output, List<String> selected) {
+ if (selected != null) {
+ config.setBytes(OUTPUT_NAME + output,
+ SerializationUtils.serialize((Serializable) selected));
+ } else {
+ config.setBytes(OUTPUT_NAME + output,
+ SerializationUtils.serialize((Serializable) new ArrayList<String>()));
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<String> getSelectedNames(String output) {
+ return (List<String>) SerializationUtils.deserialize(config.getBytes(OUTPUT_NAME + output,
+ null));
+ }
+
+ public void setNumberOfInputs(int numberOfInputs) {
+ config.setInteger(NUMBER_OF_INPUTS, numberOfInputs);
+ }
+
+ public int getNumberOfInputs() {
+ return config.getInteger(NUMBER_OF_INPUTS, 0);
+ }
+
+ public void setNumberOfOutputs(int numberOfOutputs) {
+ config.setInteger(NUMBER_OF_OUTPUTS, numberOfOutputs);
+ }
+
+ public int getNumberOfOutputs() {
+ return config.getInteger(NUMBER_OF_OUTPUTS, 0);
+ }
+
+ public void setOutputs(List<String> outputVertexNames) {
+ config.setBytes(OUTPUTS, SerializationUtils.serialize((Serializable) outputVertexNames));
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<String> getOutputs(ClassLoader cl) {
+ try {
+ return (List<String>) InstantiationUtil.readObjectFromConfig(this.config, OUTPUTS, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate outputs.");
+ }
+ }
+
+ public void setOutEdgesInOrder(List<Tuple2<String, String>> outEdgeList) {
+
+ config.setBytes(EDGES_IN_ORDER, SerializationUtils.serialize((Serializable) outEdgeList));
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<Tuple2<String, String>> getOutEdgesInOrder(ClassLoader cl) {
+ try {
+ return (List<Tuple2<String, String>>) InstantiationUtil.readObjectFromConfig(
+ this.config, EDGES_IN_ORDER, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate outputs.");
+ }
+ }
+
+ public void setInputIndex(int inputNumber, Integer inputTypeNumber) {
+ config.setInteger(INPUT_TYPE + inputNumber++, inputTypeNumber);
+ }
+
+ public int getInputIndex(int inputNumber) {
+ return config.getInteger(INPUT_TYPE + inputNumber, 0);
+ }
+
+ public void setOperatorStates(Map<String, OperatorState<?>> states) {
+ config.setBytes(OPERATOR_STATES, SerializationUtils.serialize((Serializable) states));
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<String, OperatorState<?>> getOperatorStates(ClassLoader cl) {
+ try {
+ return (Map<String, OperatorState<?>>) InstantiationUtil.readObjectFromConfig(
+ this.config, OPERATOR_STATES, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not load operator state");
+ }
+ }
+
+ public void setChainedOutputs(List<String> chainedOutputs) {
+ config.setBytes(CHAINED_OUTPUTS,
+ SerializationUtils.serialize((Serializable) chainedOutputs));
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<String> getChainedOutputs(ClassLoader cl) {
+ try {
+ return (List<String>) InstantiationUtil.readObjectFromConfig(this.config,
+ CHAINED_OUTPUTS, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate chained outputs.");
+ }
+ }
+
+ public void setTransitiveChainedTaskConfigs(Map<String, StreamConfig> chainedTaskConfigs) {
+ config.setBytes(CHAINED_TASK_CONFIG,
+ SerializationUtils.serialize((Serializable) chainedTaskConfigs));
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<String, StreamConfig> getTransitiveChainedTaskConfigs(ClassLoader cl) {
+ try {
+
+ Map<String, StreamConfig> confs = (Map<String, StreamConfig>) InstantiationUtil
+ .readObjectFromConfig(this.config, CHAINED_TASK_CONFIG, cl);
+
+ return confs == null ? new HashMap<String, StreamConfig>() : confs;
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate configuration.");
+ }
+ }
+
+ public void setChainStart() {
+ config.setBoolean(IS_CHAINED_VERTEX, true);
+ }
+
+ public boolean isChainStart() {
+ return config.getBoolean(IS_CHAINED_VERTEX, false);
+ }
+
+ @Override
+ public String toString() {
+
+ ClassLoader cl = getClass().getClassLoader();
+
+ StringBuilder builder = new StringBuilder();
+ builder.append("\n=======================");
+ builder.append("Stream Config");
+ builder.append("=======================");
+ builder.append("\nTask name: " + getTaskName());
+ builder.append("\nNumber of non-chained inputs: " + getNumberOfInputs());
+ builder.append("\nNumber of non-chained outputs: " + getNumberOfOutputs());
+ builder.append("\nOutput names: " + getOutputs(cl));
+ builder.append("\nPartitioning:");
+ for (String outputname : getOutputs(cl)) {
+ builder.append("\n\t" + outputname + ": "
+ + getPartitioner(cl, outputname).getClass().getSimpleName());
+ }
+
+ builder.append("\nChained subtasks: " + getChainedOutputs(cl));
+
+ try {
+ builder.append("\nInvokable: " + getUserInvokable(cl).getClass().getSimpleName());
+ } catch (Exception e) {
+ builder.append("\nInvokable: Missing");
+ }
+ builder.append("\nBuffer timeout: " + getBufferTimeout());
+ if (isChainStart() && getChainedOutputs(cl).size() > 0) {
+ builder.append("\n\n\n---------------------\nChained task configs\n---------------------\n");
+ builder.append(getTransitiveChainedTaskConfigs(cl)).toString();
+ }
+
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
new file mode 100644
index 0000000..b5e43af
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.compiler.plan.StreamingPlan;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.api.streamvertex.CoStreamVertex;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
+import org.apache.flink.streaming.api.streamvertex.StreamVertex;
+import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.streaming.state.OperatorState;
+import org.apache.sling.commons.json.JSONArray;
+import org.apache.sling.commons.json.JSONException;
+import org.apache.sling.commons.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Object for building Apache Flink stream processing graphs
+ */
+public class StreamGraph extends StreamingPlan {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StreamGraph.class);
+ private final static String DEAFULT_JOB_NAME = "Flink Streaming Job";
+
+ protected boolean chaining = true;
+ private String jobName = DEAFULT_JOB_NAME;
+
+ // Graph attributes
+ private Map<String, Integer> operatorParallelisms;
+ private Map<String, Long> bufferTimeouts;
+ private Map<String, List<String>> outEdgeLists;
+ private Map<String, List<Integer>> outEdgeTypes;
+ private Map<String, List<List<String>>> selectedNames;
+ private Map<String, List<String>> inEdgeLists;
+ private Map<String, List<StreamPartitioner<?>>> outputPartitioners;
+ private Map<String, String> operatorNames;
+ private Map<String, StreamInvokable<?, ?>> invokableObjects;
+ private Map<String, StreamRecordSerializer<?>> typeSerializersIn1;
+ private Map<String, StreamRecordSerializer<?>> typeSerializersIn2;
+ private Map<String, StreamRecordSerializer<?>> typeSerializersOut1;
+ private Map<String, StreamRecordSerializer<?>> typeSerializersOut2;
+ private Map<String, Class<? extends AbstractInvokable>> jobVertexClasses;
+ private Map<String, List<OutputSelector<?>>> outputSelectors;
+ private Map<String, Integer> iterationIds;
+ private Map<Integer, String> iterationIDtoHeadName;
+ private Map<Integer, String> iterationIDtoTailName;
+ private Map<String, Integer> iterationTailCount;
+ private Map<String, Long> iterationTimeouts;
+ private Map<String, Map<String, OperatorState<?>>> operatorStates;
+ private Map<String, InputFormat<String, ?>> inputFormatLists;
+
+ private Set<String> sources;
+
+ public StreamGraph() {
+
+ initGraph();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("StreamGraph created");
+ }
+ }
+
+ public void initGraph() {
+ operatorParallelisms = new HashMap<String, Integer>();
+ bufferTimeouts = new HashMap<String, Long>();
+ outEdgeLists = new HashMap<String, List<String>>();
+ outEdgeTypes = new HashMap<String, List<Integer>>();
+ selectedNames = new HashMap<String, List<List<String>>>();
+ inEdgeLists = new HashMap<String, List<String>>();
+ outputPartitioners = new HashMap<String, List<StreamPartitioner<?>>>();
+ operatorNames = new HashMap<String, String>();
+ invokableObjects = new HashMap<String, StreamInvokable<?, ?>>();
+ typeSerializersIn1 = new HashMap<String, StreamRecordSerializer<?>>();
+ typeSerializersIn2 = new HashMap<String, StreamRecordSerializer<?>>();
+ typeSerializersOut1 = new HashMap<String, StreamRecordSerializer<?>>();
+ typeSerializersOut2 = new HashMap<String, StreamRecordSerializer<?>>();
+ outputSelectors = new HashMap<String, List<OutputSelector<?>>>();
+ jobVertexClasses = new HashMap<String, Class<? extends AbstractInvokable>>();
+ iterationIds = new HashMap<String, Integer>();
+ iterationIDtoHeadName = new HashMap<Integer, String>();
+ iterationIDtoTailName = new HashMap<Integer, String>();
+ iterationTailCount = new HashMap<String, Integer>();
+ iterationTimeouts = new HashMap<String, Long>();
+ operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
+ inputFormatLists = new HashMap<String, InputFormat<String, ?>>();
+ sources = new HashSet<String>();
+ }
+
+ /**
+ * Adds a vertex to the streaming graph with the given parameters
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param invokableObject
+ * User defined operator
+ * @param inTypeInfo
+ * Input type for serialization
+ * @param outTypeInfo
+ * Output type for serialization
+ * @param operatorName
+ * Operator type
+ * @param parallelism
+ * Number of parallel instances created
+ */
+ public <IN, OUT> void addStreamVertex(String vertexName,
+ StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
+ TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
+
+ addVertex(vertexName, StreamVertex.class, invokableObject, operatorName, parallelism);
+
+ StreamRecordSerializer<IN> inSerializer = inTypeInfo != null ? new StreamRecordSerializer<IN>(
+ inTypeInfo) : null;
+ StreamRecordSerializer<OUT> outSerializer = outTypeInfo != null ? new StreamRecordSerializer<OUT>(
+ outTypeInfo) : null;
+
+ addTypeSerializers(vertexName, inSerializer, null, outSerializer, null);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Vertex: {}", vertexName);
+ }
+ }
+
+ public <IN, OUT> void addSourceVertex(String vertexName,
+ StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
+ TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
+ addStreamVertex(vertexName, invokableObject, inTypeInfo, outTypeInfo, operatorName,
+ parallelism);
+ sources.add(vertexName);
+ }
+
+ /**
+ * Adds a vertex for the iteration head to the {@link JobGraph}. The
+ * iterated values will be fed from this vertex back to the graph.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param iterationHead
+ * Id of the iteration head
+ * @param iterationID
+ * ID of iteration for multiple iterations
+ * @param parallelism
+ * Number of parallel instances created
+ * @param waitTime
+ * Max wait time for next record
+ */
+ public void addIterationHead(String vertexName, String iterationHead, Integer iterationID,
+ int parallelism, long waitTime) {
+
+ addVertex(vertexName, StreamIterationHead.class, null, null, parallelism);
+
+ chaining = false;
+
+ iterationIds.put(vertexName, iterationID);
+ iterationIDtoHeadName.put(iterationID, vertexName);
+
+ setSerializersFrom(iterationHead, vertexName);
+
+ setEdge(vertexName, iterationHead,
+ outputPartitioners.get(inEdgeLists.get(iterationHead).get(0)).get(0), 0,
+ new ArrayList<String>());
+
+ iterationTimeouts.put(iterationIDtoHeadName.get(iterationID), waitTime);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ITERATION SOURCE: {}", vertexName);
+ }
+
+ sources.add(vertexName);
+ }
+
+ /**
+ * Adds a vertex for the iteration tail to the {@link JobGraph}. The values
+ * intended to be iterated will be sent to this sink from the iteration
+ * head.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param iterationTail
+ * Id of the iteration tail
+ * @param iterationID
+ * ID of iteration for mulitple iterations
+ * @param parallelism
+ * Number of parallel instances created
+ * @param waitTime
+ * Max waiting time for next record
+ */
+ public void addIterationTail(String vertexName, String iterationTail, Integer iterationID,
+ long waitTime) {
+
+ if (bufferTimeouts.get(iterationTail) == 0) {
+ throw new RuntimeException("Buffer timeout 0 at iteration tail is not supported.");
+ }
+
+ addVertex(vertexName, StreamIterationTail.class, null, null, getParallelism(iterationTail));
+
+ iterationIds.put(vertexName, iterationID);
+ iterationIDtoTailName.put(iterationID, vertexName);
+
+ setSerializersFrom(iterationTail, vertexName);
+ iterationTimeouts.put(iterationIDtoTailName.get(iterationID), waitTime);
+
+ setParallelism(iterationIDtoHeadName.get(iterationID), getParallelism(iterationTail));
+ setBufferTimeout(iterationIDtoHeadName.get(iterationID), bufferTimeouts.get(iterationTail));
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ITERATION SINK: {}", vertexName);
+ }
+
+ }
+
+ public <IN1, IN2, OUT> void addCoTask(String vertexName,
+ CoInvokable<IN1, IN2, OUT> taskInvokableObject, TypeInformation<IN1> in1TypeInfo,
+ TypeInformation<IN2> in2TypeInfo, TypeInformation<OUT> outTypeInfo,
+ String operatorName, int parallelism) {
+
+ addVertex(vertexName, CoStreamVertex.class, taskInvokableObject, operatorName, parallelism);
+
+ addTypeSerializers(vertexName, new StreamRecordSerializer<IN1>(in1TypeInfo),
+ new StreamRecordSerializer<IN2>(in2TypeInfo), new StreamRecordSerializer<OUT>(
+ outTypeInfo), null);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CO-TASK: {}", vertexName);
+ }
+ }
+
+ /**
+ * Sets vertex parameters in the JobGraph
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param vertexClass
+ * The class of the vertex
+ * @param invokableObjectject
+ * The user defined invokable object
+ * @param operatorName
+ * Type of the user defined operator
+ * @param parallelism
+ * Number of parallel instances created
+ */
+ private void addVertex(String vertexName, Class<? extends AbstractInvokable> vertexClass,
+ StreamInvokable<?, ?> invokableObject, String operatorName, int parallelism) {
+
+ jobVertexClasses.put(vertexName, vertexClass);
+ setParallelism(vertexName, parallelism);
+ invokableObjects.put(vertexName, invokableObject);
+ operatorNames.put(vertexName, operatorName);
+ outEdgeLists.put(vertexName, new ArrayList<String>());
+ outEdgeTypes.put(vertexName, new ArrayList<Integer>());
+ selectedNames.put(vertexName, new ArrayList<List<String>>());
+ outputSelectors.put(vertexName, new ArrayList<OutputSelector<?>>());
+ inEdgeLists.put(vertexName, new ArrayList<String>());
+ outputPartitioners.put(vertexName, new ArrayList<StreamPartitioner<?>>());
+ iterationTailCount.put(vertexName, 0);
+ }
+
+ /**
+ * Connects two vertices in the JobGraph using the selected partitioner
+ * settings
+ *
+ * @param upStreamVertexName
+ * Name of the upstream(output) vertex
+ * @param downStreamVertexName
+ * Name of the downstream(input) vertex
+ * @param partitionerObject
+ * Partitioner object
+ * @param typeNumber
+ * Number of the type (used at co-functions)
+ * @param outputNames
+ * User defined names of the out edge
+ */
+ public void setEdge(String upStreamVertexName, String downStreamVertexName,
+ StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames) {
+ outEdgeLists.get(upStreamVertexName).add(downStreamVertexName);
+ outEdgeTypes.get(upStreamVertexName).add(typeNumber);
+ inEdgeLists.get(downStreamVertexName).add(upStreamVertexName);
+ outputPartitioners.get(upStreamVertexName).add(partitionerObject);
+ selectedNames.get(upStreamVertexName).add(outputNames);
+ }
+
+ private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
+ StreamRecordSerializer<?> in2, StreamRecordSerializer<?> out1,
+ StreamRecordSerializer<?> out2) {
+ typeSerializersIn1.put(vertexName, in1);
+ typeSerializersIn2.put(vertexName, in2);
+ typeSerializersOut1.put(vertexName, out1);
+ typeSerializersOut2.put(vertexName, out2);
+ }
+
+ /**
+ * Sets the number of parallel instances created for the given vertex.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param parallelism
+ * Number of parallel instances created
+ */
+ public void setParallelism(String vertexName, int parallelism) {
+ operatorParallelisms.put(vertexName, parallelism);
+ }
+
+ public int getParallelism(String vertexName) {
+ return operatorParallelisms.get(vertexName);
+ }
+
+ /**
+ * Sets the input format for the given vertex.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param inputFormat
+ * input format of the file source associated with the given
+ * vertex
+ */
+ public void setInputFormat(String vertexName, InputFormat<String, ?> inputFormat) {
+ inputFormatLists.put(vertexName, inputFormat);
+ }
+
+ public void setBufferTimeout(String vertexName, long bufferTimeout) {
+ this.bufferTimeouts.put(vertexName, bufferTimeout);
+ }
+
+ public long getBufferTimeout(String vertexName) {
+ return this.bufferTimeouts.get(vertexName);
+ }
+
+ public void addOperatorState(String veretxName, String stateName, OperatorState<?> state) {
+ Map<String, OperatorState<?>> states = operatorStates.get(veretxName);
+ if (states == null) {
+ states = new HashMap<String, OperatorState<?>>();
+ states.put(stateName, state);
+ } else {
+ if (states.containsKey(stateName)) {
+ throw new RuntimeException("State has already been registered with this name: "
+ + stateName);
+ } else {
+ states.put(stateName, state);
+ }
+ }
+ operatorStates.put(veretxName, states);
+ }
+
+ /**
+ * Sets a user defined {@link OutputSelector} for the given operator. Used
+ * for directed emits.
+ *
+ * @param vertexName
+ * Name of the vertex for which the output selector will be set
+ * @param outputSelector
+ * The user defined output selector.
+ */
+ public <T> void setOutputSelector(String vertexName, OutputSelector<T> outputSelector) {
+ outputSelectors.get(vertexName).add(outputSelector);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Outputselector set for {}", vertexName);
+ }
+
+ }
+
+ public <IN, OUT> void setInvokable(String vertexName, StreamInvokable<IN, OUT> invokableObject) {
+ invokableObjects.put(vertexName, invokableObject);
+ }
+
+ public <OUT> void setOutType(String id, TypeInformation<OUT> outType) {
+ StreamRecordSerializer<OUT> serializer = new StreamRecordSerializer<OUT>(outType);
+ typeSerializersOut1.put(id, serializer);
+ }
+
+ public StreamInvokable<?, ?> getInvokable(String vertexName) {
+ return invokableObjects.get(vertexName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <OUT> StreamRecordSerializer<OUT> getOutSerializer1(String vertexName) {
+ return (StreamRecordSerializer<OUT>) typeSerializersOut1.get(vertexName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <OUT> StreamRecordSerializer<OUT> getOutSerializer2(String vertexName) {
+ return (StreamRecordSerializer<OUT>) typeSerializersOut2.get(vertexName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <IN> StreamRecordSerializer<IN> getInSerializer1(String vertexName) {
+ return (StreamRecordSerializer<IN>) typeSerializersIn1.get(vertexName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <IN> StreamRecordSerializer<IN> getInSerializer2(String vertexName) {
+ return (StreamRecordSerializer<IN>) typeSerializersIn2.get(vertexName);
+ }
+
+ /**
+ * Sets TypeSerializerWrapper from one vertex to another, used with some
+ * sinks.
+ *
+ * @param from
+ * from
+ * @param to
+ * to
+ */
+ public void setSerializersFrom(String from, String to) {
+ operatorNames.put(to, operatorNames.get(from));
+
+ typeSerializersIn1.put(to, typeSerializersOut1.get(from));
+ typeSerializersIn2.put(to, typeSerializersOut2.get(from));
+ typeSerializersOut1.put(to, typeSerializersOut1.get(from));
+ typeSerializersOut2.put(to, typeSerializersOut2.get(from));
+ }
+
+ /**
+ * Gets the assembled {@link JobGraph} and adds a default name for it.
+ */
+ public JobGraph getJobGraph() {
+ return getJobGraph(jobName);
+ }
+
+ /**
+ * Gets the assembled {@link JobGraph} and adds a user specified name for
+ * it.
+ *
+ * @param jobGraphName
+ * name of the jobGraph
+ */
+ public JobGraph getJobGraph(String jobGraphName) {
+
+ this.jobName = jobGraphName;
+ StreamingJobGraphGenerator optimizer = new StreamingJobGraphGenerator(this);
+
+ return optimizer.createJobGraph(jobGraphName);
+ }
+
+ public void setJobName(String jobName) {
+ this.jobName = jobName;
+ }
+
+ public void setChaining(boolean chaining) {
+ this.chaining = chaining;
+ }
+
+ public Collection<String> getSources() {
+ return sources;
+ }
+
+ public List<String> getOutEdges(String vertexName) {
+ return outEdgeLists.get(vertexName);
+ }
+
+ public List<String> getInEdges(String vertexName) {
+ return inEdgeLists.get(vertexName);
+ }
+
+ public List<Integer> getOutEdgeTypes(String vertexName) {
+
+ return outEdgeTypes.get(vertexName);
+ }
+
+ public StreamPartitioner<?> getOutPartitioner(String upStreamVertex, String downStreamVertex) {
+ return outputPartitioners.get(upStreamVertex).get(
+ outEdgeLists.get(upStreamVertex).indexOf(downStreamVertex));
+ }
+
+ public List<String> getSelectedNames(String upStreamVertex, String downStreamVertex) {
+
+ return selectedNames.get(upStreamVertex).get(
+ outEdgeLists.get(upStreamVertex).indexOf(downStreamVertex));
+ }
+
+ public Collection<Integer> getIterationIDs() {
+ return new HashSet<Integer>(iterationIds.values());
+ }
+
+ public String getIterationTail(int iterID) {
+ return iterationIDtoTailName.get(iterID);
+ }
+
+ public String getIterationHead(int iterID) {
+ return iterationIDtoHeadName.get(iterID);
+ }
+
+ public Class<? extends AbstractInvokable> getJobVertexClass(String vertexName) {
+ return jobVertexClasses.get(vertexName);
+ }
+
+ public InputFormat<String, ?> getInputFormat(String vertexName) {
+ return inputFormatLists.get(vertexName);
+ }
+
+ public List<OutputSelector<?>> getOutputSelector(String vertexName) {
+ return outputSelectors.get(vertexName);
+ }
+
+ public Map<String, OperatorState<?>> getState(String vertexName) {
+ return operatorStates.get(vertexName);
+ }
+
+ public Integer getIterationID(String vertexName) {
+ return iterationIds.get(vertexName);
+ }
+
+ public long getIterationTimeout(String vertexName) {
+ return iterationTimeouts.get(vertexName);
+ }
+
+ public String getOperatorName(String vertexName) {
+ return operatorNames.get(vertexName);
+ }
+
+ @Override
+ public String getStreamingPlanAsJSON() {
+
+ try {
+ JSONObject json = new JSONObject();
+ JSONArray nodes = new JSONArray();
+
+ json.put("nodes", nodes);
+
+ for (String id : operatorNames.keySet()) {
+ JSONObject node = new JSONObject();
+ nodes.put(node);
+
+ node.put("id", Integer.valueOf(id));
+ node.put("type", getOperatorName(id));
+
+ if (sources.contains(id)) {
+ node.put("pact", "Data Source");
+ } else {
+ node.put("pact", "Data Stream");
+ }
+
+ node.put("contents", getOperatorName(id) + " at "
+ + getInvokable(id).getUserFunction().getClass().getSimpleName());
+ node.put("parallelism", getParallelism(id));
+
+ int numIn = getInEdges(id).size();
+ if (numIn > 0) {
+
+ JSONArray inputs = new JSONArray();
+ node.put("predecessors", inputs);
+
+ for (int i = 0; i < numIn; i++) {
+
+ String inID = getInEdges(id).get(i);
+
+ JSONObject input = new JSONObject();
+ inputs.put(input);
+
+ input.put("id", Integer.valueOf(inID));
+ input.put("ship_strategy", getOutPartitioner(inID, id).getStrategy());
+ if (i == 0) {
+ input.put("side", "first");
+ } else if (i == 1) {
+ input.put("side", "second");
+ }
+ }
+ }
+
+ }
+ return json.toString();
+ } catch (JSONException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("JSON plan creation failed: {}", e);
+ }
+ return "";
+ }
+
+ }
+
+ @Override
+ public void dumpStreamingPlanAsJSON(File file) throws IOException {
+ PrintWriter pw = null;
+ try {
+ pw = new PrintWriter(new FileOutputStream(file), false);
+ pw.write(getStreamingPlanAsJSON());
+ pw.flush();
+
+ } finally {
+ if (pw != null) {
+ pw.close();
+ }
+ }
+ }
+}