You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2015/11/12 21:09:22 UTC

[7/8] flink git commit: [FLINK-2901] Move PythonAPI to flink-libraries

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/Sender.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/Sender.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/Sender.java
new file mode 100644
index 0000000..b897f12
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/Sender.java
@@ -0,0 +1,411 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.flink.python.api.streaming;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.java.tuple.Tuple;
+import static org.apache.flink.python.api.PythonPlanBinder.FLINK_TMP_DATA_DIR;
+import static org.apache.flink.python.api.PythonPlanBinder.MAPPED_FILE_SIZE;
+
+/**
+ * General-purpose class to write data to memory-mapped files.
+ */
+public class Sender implements Serializable {
+	public static final byte TYPE_TUPLE = (byte) 11;
+	public static final byte TYPE_BOOLEAN = (byte) 10;
+	public static final byte TYPE_BYTE = (byte) 9;
+	public static final byte TYPE_SHORT = (byte) 8;
+	public static final byte TYPE_INTEGER = (byte) 7;
+	public static final byte TYPE_LONG = (byte) 6;
+	public static final byte TYPE_DOUBLE = (byte) 4;
+	public static final byte TYPE_FLOAT = (byte) 5;
+	public static final byte TYPE_CHAR = (byte) 3;
+	public static final byte TYPE_STRING = (byte) 2;
+	public static final byte TYPE_BYTES = (byte) 1;
+	public static final byte TYPE_NULL = (byte) 0;
+
+	private final AbstractRichFunction function;
+
+	private File outputFile;
+	private RandomAccessFile outputRAF;
+	private FileChannel outputChannel;
+	private MappedByteBuffer fileBuffer;
+
+	private final ByteBuffer[] saved = new ByteBuffer[2];
+
+	private final Serializer[] serializer = new Serializer[2];
+
+	public Sender(AbstractRichFunction function) {
+		this.function = function;
+	}
+
+	//=====Setup========================================================================================================
+	public void open(String path) throws IOException {
+		setupMappedFile(path);
+	}
+
+	private void setupMappedFile(String outputFilePath) throws FileNotFoundException, IOException {
+		File x = new File(FLINK_TMP_DATA_DIR);
+		x.mkdirs();
+
+		outputFile = new File(outputFilePath);
+		if (outputFile.exists()) {
+			outputFile.delete();
+		}
+		outputFile.createNewFile();
+		outputRAF = new RandomAccessFile(outputFilePath, "rw");
+		outputRAF.setLength(MAPPED_FILE_SIZE);
+		outputRAF.seek(MAPPED_FILE_SIZE - 1);
+		outputRAF.writeByte(0);
+		outputRAF.seek(0);
+		outputChannel = outputRAF.getChannel();
+		fileBuffer = outputChannel.map(FileChannel.MapMode.READ_WRITE, 0, MAPPED_FILE_SIZE);
+	}
+
+	public void close() throws IOException {
+		closeMappedFile();
+	}
+
+	private void closeMappedFile() throws IOException {
+		outputChannel.close();
+		outputRAF.close();
+	}
+
+	/**
+	 * Resets this object to the post-configuration state.
+	 */
+	public void reset() {
+		serializer[0] = null;
+		serializer[1] = null;
+		fileBuffer.clear();
+	}
+
+	//=====Serialization================================================================================================
+	/**
+	 * Writes a single record to the memory-mapped file. This method does NOT take care of synchronization. The user
+	 * must guarantee that the file may be written to before calling this method. This method essentially reserves the
+	 * whole buffer for one record. As such it imposes some performance restrictions and should only be used when
+	 * absolutely necessary.
+	 *
+	 * @param value record to send
+	 * @return size of the written buffer
+	 * @throws IOException
+	 */
+	public int sendRecord(Object value) throws IOException {
+		fileBuffer.clear();
+		int group = 0;
+
+		serializer[group] = getSerializer(value);
+		ByteBuffer bb = serializer[group].serialize(value);
+		if (bb.remaining() > MAPPED_FILE_SIZE) {
+			throw new RuntimeException("Serialized object does not fit into a single buffer.");
+		}
+		fileBuffer.put(bb);
+
+		int size = fileBuffer.position();
+
+		reset();
+		return size;
+	}
+
+	public boolean hasRemaining(int group) {
+		return saved[group] != null;
+	}
+
+	/**
+	 * Extracts records from an iterator and writes them to the memory-mapped file. This method assumes that all values
+	 * in the iterator are of the same type. This method does NOT take care of synchronization. The caller must
+	 * guarantee that the file may be written to before calling this method.
+	 *
+	 * @param i iterator containing records
+	 * @param group group to which the iterator belongs, most notably used by CoGroup-functions.
+	 * @return size of the written buffer
+	 * @throws IOException
+	 */
+	public int sendBuffer(Iterator i, int group) throws IOException {
+		fileBuffer.clear();
+
+		Object value;
+		ByteBuffer bb;
+		if (serializer[group] == null) {
+			value = i.next();
+			serializer[group] = getSerializer(value);
+			bb = serializer[group].serialize(value);
+			if (bb.remaining() > MAPPED_FILE_SIZE) {
+				throw new RuntimeException("Serialized object does not fit into a single buffer.");
+			}
+			fileBuffer.put(bb);
+
+		}
+		if (saved[group] != null) {
+			fileBuffer.put(saved[group]);
+			saved[group] = null;
+		}
+		while (i.hasNext() && saved[group] == null) {
+			value = i.next();
+			bb = serializer[group].serialize(value);
+			if (bb.remaining() > MAPPED_FILE_SIZE) {
+				throw new RuntimeException("Serialized object does not fit into a single buffer.");
+			}
+			if (bb.remaining() <= fileBuffer.remaining()) {
+				fileBuffer.put(bb);
+			} else {
+				saved[group] = bb;
+			}
+		}
+
+		int size = fileBuffer.position();
+		return size;
+	}
+
+	private enum SupportedTypes {
+		TUPLE, BOOLEAN, BYTE, BYTES, CHARACTER, SHORT, INTEGER, LONG, FLOAT, DOUBLE, STRING, OTHER, NULL
+	}
+
+	//=====Serializer===================================================================================================
+	private Serializer getSerializer(Object value) throws IOException {
+		String className = value.getClass().getSimpleName().toUpperCase();
+		if (className.startsWith("TUPLE")) {
+			className = "TUPLE";
+		}
+		if (className.startsWith("BYTE[]")) {
+			className = "BYTES";
+		}
+		SupportedTypes type = SupportedTypes.valueOf(className);
+		switch (type) {
+			case TUPLE:
+				fileBuffer.put(TYPE_TUPLE);
+				fileBuffer.putInt(((Tuple) value).getArity());
+				return new TupleSerializer((Tuple) value);
+			case BOOLEAN:
+				fileBuffer.put(TYPE_BOOLEAN);
+				return new BooleanSerializer();
+			case BYTE:
+				fileBuffer.put(TYPE_BYTE);
+				return new ByteSerializer();
+			case BYTES:
+				fileBuffer.put(TYPE_BYTES);
+				return new BytesSerializer();
+			case CHARACTER:
+				fileBuffer.put(TYPE_CHAR);
+				return new CharSerializer();
+			case SHORT:
+				fileBuffer.put(TYPE_SHORT);
+				return new ShortSerializer();
+			case INTEGER:
+				fileBuffer.put(TYPE_INTEGER);
+				return new IntSerializer();
+			case LONG:
+				fileBuffer.put(TYPE_LONG);
+				return new LongSerializer();
+			case STRING:
+				fileBuffer.put(TYPE_STRING);
+				return new StringSerializer();
+			case FLOAT:
+				fileBuffer.put(TYPE_FLOAT);
+				return new FloatSerializer();
+			case DOUBLE:
+				fileBuffer.put(TYPE_DOUBLE);
+				return new DoubleSerializer();
+			case NULL:
+				fileBuffer.put(TYPE_NULL);
+				return new NullSerializer();
+			default:
+				throw new IllegalArgumentException("Unknown Type encountered: " + type);
+		}
+	}
+
+	private abstract class Serializer<T> {
+		protected ByteBuffer buffer;
+
+		public Serializer(int capacity) {
+			buffer = ByteBuffer.allocate(capacity);
+		}
+
+		public ByteBuffer serialize(T value) {
+			buffer.clear();
+			serializeInternal(value);
+			buffer.flip();
+			return buffer;
+		}
+
+		public abstract void serializeInternal(T value);
+	}
+
+	private class ByteSerializer extends Serializer<Byte> {
+		public ByteSerializer() {
+			super(1);
+		}
+
+		@Override
+		public void serializeInternal(Byte value) {
+			buffer.put(value);
+		}
+	}
+
+	private class BooleanSerializer extends Serializer<Boolean> {
+		public BooleanSerializer() {
+			super(1);
+		}
+
+		@Override
+		public void serializeInternal(Boolean value) {
+			buffer.put(value ? (byte) 1 : (byte) 0);
+		}
+	}
+
+	private class CharSerializer extends Serializer<Character> {
+		public CharSerializer() {
+			super(4);
+		}
+
+		@Override
+		public void serializeInternal(Character value) {
+			buffer.put((value + "").getBytes());
+		}
+	}
+
+	private class ShortSerializer extends Serializer<Short> {
+		public ShortSerializer() {
+			super(2);
+		}
+
+		@Override
+		public void serializeInternal(Short value) {
+			buffer.putShort(value);
+		}
+	}
+
+	private class IntSerializer extends Serializer<Integer> {
+		public IntSerializer() {
+			super(4);
+		}
+
+		@Override
+		public void serializeInternal(Integer value) {
+			buffer.putInt(value);
+		}
+	}
+
+	private class LongSerializer extends Serializer<Long> {
+		public LongSerializer() {
+			super(8);
+		}
+
+		@Override
+		public void serializeInternal(Long value) {
+			buffer.putLong(value);
+		}
+	}
+
+	private class StringSerializer extends Serializer<String> {
+		public StringSerializer() {
+			super(0);
+		}
+
+		@Override
+		public void serializeInternal(String value) {
+			byte[] bytes = value.getBytes();
+			buffer = ByteBuffer.allocate(bytes.length + 4);
+			buffer.putInt(bytes.length);
+			buffer.put(bytes);
+		}
+	}
+
+	private class FloatSerializer extends Serializer<Float> {
+		public FloatSerializer() {
+			super(4);
+		}
+
+		@Override
+		public void serializeInternal(Float value) {
+			buffer.putFloat(value);
+		}
+	}
+
+	private class DoubleSerializer extends Serializer<Double> {
+		public DoubleSerializer() {
+			super(8);
+		}
+
+		@Override
+		public void serializeInternal(Double value) {
+			buffer.putDouble(value);
+		}
+	}
+
+	private class NullSerializer extends Serializer<Object> {
+		public NullSerializer() {
+			super(0);
+		}
+
+		@Override
+		public void serializeInternal(Object value) {
+		}
+	}
+
+	private class BytesSerializer extends Serializer<byte[]> {
+		public BytesSerializer() {
+			super(0);
+		}
+
+		@Override
+		public void serializeInternal(byte[] value) {
+			buffer = ByteBuffer.allocate(4 + value.length);
+			buffer.putInt(value.length);
+			buffer.put(value);
+		}
+	}
+
+	private class TupleSerializer extends Serializer<Tuple> {
+		private final Serializer[] serializer;
+		private final List<ByteBuffer> buffers;
+
+		public TupleSerializer(Tuple value) throws IOException {
+			super(0);
+			serializer = new Serializer[value.getArity()];
+			buffers = new ArrayList();
+			for (int x = 0; x < serializer.length; x++) {
+				serializer[x] = getSerializer(value.getField(x));
+			}
+		}
+
+		@Override
+		public void serializeInternal(Tuple value) {
+			int length = 0;
+			for (int x = 0; x < serializer.length; x++) {
+				serializer[x].buffer.clear();
+				serializer[x].serializeInternal(value.getField(x));
+				length += serializer[x].buffer.position();
+				buffers.add(serializer[x].buffer);
+			}
+			buffer = ByteBuffer.allocate(length);
+			for (ByteBuffer b : buffers) {
+				b.flip();
+				buffer.put(b);
+			}
+			buffers.clear();
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/StreamPrinter.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/StreamPrinter.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/StreamPrinter.java
new file mode 100644
index 0000000..e42364b
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/StreamPrinter.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.flink.python.api.streaming;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * Simple utility class to print all contents of an inputstream to stdout.
+ */
+public class StreamPrinter extends Thread {
+	private final BufferedReader reader;
+	private final boolean wrapInException;
+	private StringBuilder msg;
+
+	public StreamPrinter(InputStream stream) {
+		this(stream, false, null);
+	}
+
+	public StreamPrinter(InputStream stream, boolean wrapInException, StringBuilder msg) {
+		this.reader = new BufferedReader(new InputStreamReader(stream));
+		this.wrapInException = wrapInException;
+		this.msg = msg;
+	}
+
+	@Override
+	public void run() {
+		String line;
+		try {
+			if (wrapInException) {
+				while ((line = reader.readLine()) != null) {
+					msg.append("\n" + line);
+				}
+			} else {
+				while ((line = reader.readLine()) != null) {
+					System.out.println(line);
+					System.out.flush();
+				}
+			}
+		} catch (IOException ex) {
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/__init__.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/__init__.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/__init__.py
new file mode 100644
index 0000000..d35bf39
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/__init__.py
@@ -0,0 +1,17 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/__init__.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/__init__.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/__init__.py
new file mode 100644
index 0000000..d35bf39
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/__init__.py
@@ -0,0 +1,17 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Collector.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Collector.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Collector.py
new file mode 100644
index 0000000..bf35756
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Collector.py
@@ -0,0 +1,157 @@
+# ###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from struct import pack
+import sys
+
+from flink.connection.Constants import Types
+
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+if PY2:
+    stringtype = basestring
+else:
+    stringtype = str
+
+
+class Collector(object):
+    def __init__(self, con):
+        self._connection = con
+        self._serializer = None
+
+    def _close(self):
+        self._connection.send_end_signal()
+
+    def collect(self, value):
+        self._serializer = _get_serializer(self._connection.write, value)
+        self.collect = self._collect
+        self.collect(value)
+
+    def _collect(self, value):
+        self._connection.write(self._serializer.serialize(value))
+
+
+def _get_serializer(write, value):
+    if isinstance(value, (list, tuple)):
+        write(Types.TYPE_TUPLE)
+        write(pack(">I", len(value)))
+        return TupleSerializer(write, value)
+    elif value is None:
+        write(Types.TYPE_NULL)
+        return NullSerializer()
+    elif isinstance(value, stringtype):
+        write(Types.TYPE_STRING)
+        return StringSerializer()
+    elif isinstance(value, bool):
+        write(Types.TYPE_BOOLEAN)
+        return BooleanSerializer()
+    elif isinstance(value, int) or PY2 and isinstance(value, long):
+        write(Types.TYPE_LONG)
+        return LongSerializer()
+    elif isinstance(value, bytearray):
+        write(Types.TYPE_BYTES)
+        return ByteArraySerializer()
+    elif isinstance(value, float):
+        write(Types.TYPE_DOUBLE)
+        return FloatSerializer()
+    else:
+        raise Exception("Unsupported Type encountered.")
+
+
+class TupleSerializer(object):
+    def __init__(self, write, value):
+        self.serializer = [_get_serializer(write, field) for field in value]
+
+    def serialize(self, value):
+        bits = []
+        for i in range(len(value)):
+            bits.append(self.serializer[i].serialize(value[i]))
+        return b"".join(bits)
+
+
+class BooleanSerializer(object):
+    def serialize(self, value):
+        return pack(">?", value)
+
+
+class FloatSerializer(object):
+    def serialize(self, value):
+        return pack(">d", value)
+
+
+class LongSerializer(object):
+    def serialize(self, value):
+        return pack(">q", value)
+
+
+class ByteArraySerializer(object):
+    def serialize(self, value):
+        value = bytes(value)
+        return pack(">I", len(value)) + value
+
+
+class StringSerializer(object):
+    def serialize(self, value):
+        value = value.encode("utf-8")
+        return pack(">I", len(value)) + value
+
+
+class NullSerializer(object):
+    def serialize(self, value):
+        return b""
+
+
+class TypedCollector(object):
+    def __init__(self, con):
+        self._connection = con
+
+    def collect(self, value):
+        if not isinstance(value, (list, tuple)):
+            self._send_field(value)
+        else:
+            self._connection.write(Types.TYPE_TUPLE)
+            meta = pack(">I", len(value))
+            self._connection.write(bytes([meta[3]]) if PY3 else meta[3])
+            for field in value:
+                self.collect(field)
+
+    def _send_field(self, value):
+        if value is None:
+            self._connection.write(Types.TYPE_NULL)
+        elif isinstance(value, stringtype):
+            value = value.encode("utf-8")
+            size = pack(">I", len(value))
+            self._connection.write(b"".join([Types.TYPE_STRING, size, value]))
+        elif isinstance(value, bytes):
+            size = pack(">I", len(value))
+            self._connection.write(b"".join([Types.TYPE_BYTES, size, value]))
+        elif isinstance(value, bool):
+            data = pack(">?", value)
+            self._connection.write(b"".join([Types.TYPE_BOOLEAN, data]))
+        elif isinstance(value, int) or PY2 and isinstance(value, long):
+            data = pack(">q", value)
+            self._connection.write(b"".join([Types.TYPE_LONG, data]))
+        elif isinstance(value, float):
+            data = pack(">d", value)
+            self._connection.write(b"".join([Types.TYPE_DOUBLE, data]))
+        elif isinstance(value, bytearray):
+            value = bytes(value)
+            size = pack(">I", len(value))
+            self._connection.write(b"".join([Types.TYPE_BYTES, size, value]))
+        else:
+            raise Exception("Unsupported Type encountered.")
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Connection.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Connection.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Connection.py
new file mode 100644
index 0000000..988bf25
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Connection.py
@@ -0,0 +1,166 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import mmap
+import socket as SOCKET
+from struct import pack, unpack
+from collections import deque
+import sys
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+MAPPED_FILE_SIZE = 1024 * 1024 * 64
+
+SIGNAL_REQUEST_BUFFER = b"\x00\x00\x00\x00"
+SIGNAL_REQUEST_BUFFER_G0 = b"\xFF\xFF\xFF\xFD"
+SIGNAL_REQUEST_BUFFER_G1 = b"\xFF\xFF\xFF\xFC"
+SIGNAL_FINISHED = b"\xFF\xFF\xFF\xFF"
+
+if PY2:
+    SIGNAL_WAS_LAST = "\x20"
+else:
+    SIGNAL_WAS_LAST = 32
+
+
+class OneWayBusyBufferingMappedFileConnection(object):
+    def __init__(self, output_path):
+        self._output_file = open(output_path, "rb+")
+        self._file_output_buffer = mmap.mmap(self._output_file.fileno(), MAPPED_FILE_SIZE, mmap.MAP_SHARED, mmap.ACCESS_WRITE)
+
+        self._out = deque()
+        self._out_size = 0
+
+        self._offset_limit = MAPPED_FILE_SIZE - 1024 * 1024 * 3
+
+    def write(self, msg):
+        self._out.append(msg)
+        self._out_size += len(msg)
+        if self._out_size > self._offset_limit:
+            self._write_buffer()
+
+    def _write_buffer(self):
+        self._file_output_buffer.seek(1, 0)
+        self._file_output_buffer.write(b"".join(self._out))
+        self._file_output_buffer.seek(0, 0)
+        self._file_output_buffer.write(b'\x01')
+
+
+class BufferingTCPMappedFileConnection(object):
+    def __init__(self, input_file, output_file, port):
+        self._input_file = open(input_file, "rb+")
+        self._output_file = open(output_file, "rb+")
+        self._file_input_buffer = mmap.mmap(self._input_file.fileno(), MAPPED_FILE_SIZE, mmap.MAP_SHARED, mmap.ACCESS_READ)
+        self._file_output_buffer = mmap.mmap(self._output_file.fileno(), MAPPED_FILE_SIZE, mmap.MAP_SHARED, mmap.ACCESS_WRITE)
+        self._socket = SOCKET.socket(family=SOCKET.AF_INET, type=SOCKET.SOCK_STREAM)
+        self._socket.connect((SOCKET.gethostbyname("localhost"), port))
+
+        self._out = deque()
+        self._out_size = 0
+
+        self._input = b""
+        self._input_offset = 0
+        self._input_size = 0
+        self._was_last = False
+
+    def write(self, msg):
+        length = len(msg)
+        if length > MAPPED_FILE_SIZE:
+            raise Exception("Serialized object does not fit into a single buffer.")
+        tmp = self._out_size + length
+        if tmp > MAPPED_FILE_SIZE:
+            self._write_buffer()
+            self.write(msg)
+        else:
+            self._out.append(msg)
+            self._out_size = tmp
+
+    def _write_buffer(self):
+        self._file_output_buffer.seek(0, 0)
+        self._file_output_buffer.write(b"".join(self._out))
+        self._socket.send(pack(">i", self._out_size))
+        self._out.clear()
+        self._out_size = 0
+        self._socket.recv(1, SOCKET.MSG_WAITALL)
+
+    def read(self, des_size, ignored=None):
+        if self._input_size == self._input_offset:
+            self._read_buffer()
+        old_offset = self._input_offset
+        self._input_offset += des_size
+        return self._input[old_offset:self._input_offset]
+
+    def _read_buffer(self):
+        self._socket.send(SIGNAL_REQUEST_BUFFER)
+        self._file_input_buffer.seek(0, 0)
+        self._input_offset = 0
+        meta_size = self._socket.recv(5, SOCKET.MSG_WAITALL)
+        self._input_size = unpack(">I", meta_size[:4])[0]
+        self._was_last = meta_size[4] == SIGNAL_WAS_LAST
+        self._input = self._file_input_buffer.read(self._input_size)
+
+    def send_end_signal(self):
+        if self._out_size:
+            self._write_buffer()
+        self._socket.send(SIGNAL_FINISHED)
+
+    def has_next(self, ignored=None):
+        return not self._was_last or not self._input_size == self._input_offset
+
+    def reset(self):
+        self._was_last = False
+        self._input_size = 0
+        self._input_offset = 0
+        self._input = b""
+
+
+class TwinBufferingTCPMappedFileConnection(BufferingTCPMappedFileConnection):
+    def __init__(self, input_file, output_file, port):
+        super(TwinBufferingTCPMappedFileConnection, self).__init__(input_file, output_file, port)
+        self._input = [b"", b""]
+        self._input_offset = [0, 0]
+        self._input_size = [0, 0]
+        self._was_last = [False, False]
+
+    def read(self, des_size, group):
+        if self._input_size[group] == self._input_offset[group]:
+            self._read_buffer(group)
+        old_offset = self._input_offset[group]
+        self._input_offset[group] += des_size
+        return self._input[group][old_offset:self._input_offset[group]]
+
+    def _read_buffer(self, group):
+        if group:
+            self._socket.send(SIGNAL_REQUEST_BUFFER_G1)
+        else:
+            self._socket.send(SIGNAL_REQUEST_BUFFER_G0)
+        self._file_input_buffer.seek(0, 0)
+        self._input_offset[group] = 0
+        meta_size = self._socket.recv(5, SOCKET.MSG_WAITALL)
+        self._input_size[group] = unpack(">I", meta_size[:4])[0]
+        self._was_last[group] = meta_size[4] == SIGNAL_WAS_LAST
+        self._input[group] = self._file_input_buffer.read(self._input_size[group])
+
+    def has_next(self, group):
+        return not self._was_last[group] or not self._input_size[group] == self._input_offset[group]
+
+    def reset(self):
+        self._was_last = [False, False]
+        self._input_size = [0, 0]
+        self._input_offset = [0, 0]
+        self._input = [b"", b""]
+
+

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Constants.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Constants.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Constants.py
new file mode 100644
index 0000000..0ca2232
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Constants.py
@@ -0,0 +1,31 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+class Types(object):
+    TYPE_TUPLE = b'\x0B'
+    TYPE_BOOLEAN = b'\x0A'
+    TYPE_BYTE = b'\x09'
+    TYPE_SHORT = b'\x08'
+    TYPE_INTEGER = b'\x07'
+    TYPE_LONG = b'\x06'
+    TYPE_DOUBLE = b'\x04'
+    TYPE_FLOAT = b'\x05'
+    TYPE_STRING = b'\x02'
+    TYPE_NULL = b'\x00'
+    TYPE_BYTES = b'\x01'

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Iterator.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Iterator.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Iterator.py
new file mode 100644
index 0000000..fb0e26d
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Iterator.py
@@ -0,0 +1,327 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from struct import unpack
+from collections import deque
+
+try:
+    import _abcoll as defIter
+except:
+    import _collections_abc as defIter
+
+from flink.connection.Constants import Types
+
+
+class ListIterator(defIter.Iterator):
+    def __init__(self, values):
+        super(ListIterator, self).__init__()
+        self._values = deque(values)
+
+    def __next__(self):
+        return self.next()
+
+    def next(self):
+        if self.has_next():
+            return self._values.popleft()
+        else:
+            raise StopIteration
+
+    def has_next(self):
+        return self._values
+
+
+class GroupIterator(defIter.Iterator):
+    def __init__(self, iterator, keys=None):
+        super(GroupIterator, self).__init__()
+        self.iterator = iterator
+        self.key = None
+        self.keys = keys
+        if self.keys is None:
+            self._extract_keys = self._extract_keys_id
+        self.cur = None
+        self.empty = False
+
+    def _init(self):
+        if self.iterator.has_next():
+            self.empty = False
+            self.cur = self.iterator.next()
+            self.key = self._extract_keys(self.cur)
+        else:
+            self.empty = True
+
+    def __next__(self):
+        return self.next()
+
+    def next(self):
+        if self.has_next():
+            tmp = self.cur
+            if self.iterator.has_next():
+                self.cur = self.iterator.next()
+                if self.key != self._extract_keys(self.cur):
+                    self.empty = True
+            else:
+                self.cur = None
+                self.empty = True
+            return tmp
+        else:
+            raise StopIteration
+
+    def has_next(self):
+        if self.empty:
+            return False
+        return self.key == self._extract_keys(self.cur)
+
+    def has_group(self):
+        return self.cur is not None
+
+    def next_group(self):
+        self.key = self._extract_keys(self.cur)
+        self.empty = False
+
+    def _extract_keys(self, x):
+        return [x[k] for k in self.keys]
+
+    def _extract_keys_id(self, x):
+        return x
+
+
+class CoGroupIterator(object):
+    NONE_REMAINED = 1
+    FIRST_REMAINED = 2
+    SECOND_REMAINED = 3
+    FIRST_EMPTY = 4
+    SECOND_EMPTY = 5
+
+    def __init__(self, c1, c2, k1, k2):
+        self.i1 = GroupIterator(c1, k1)
+        self.i2 = GroupIterator(c2, k2)
+        self.p1 = None
+        self.p2 = None
+        self.match = None
+        self.key = None
+
+    def _init(self):
+        self.i1._init()
+        self.i2._init()
+
+    def next(self):
+        first_empty = True
+        second_empty = True
+
+        if self.match != CoGroupIterator.FIRST_EMPTY:
+            if self.match == CoGroupIterator.FIRST_REMAINED:
+                first_empty = False
+            else:
+                if self.i1.has_group():
+                    self.i1.next_group()
+                    self.key = self.i1.key
+                    first_empty = False
+
+        if self.match != CoGroupIterator.SECOND_EMPTY:
+            if self.match == CoGroupIterator.SECOND_REMAINED:
+                second_empty = False
+            else:
+                if self.i2.has_group():
+                    self.i2.next_group()
+                    second_empty = False
+
+        if first_empty and second_empty:
+            return False
+        elif first_empty and (not second_empty):
+            self.p1 = DummyIterator()
+            self.p2 = self.i2
+            self.match = CoGroupIterator.FIRST_EMPTY
+            return True
+        elif (not first_empty) and second_empty:
+            self.p1 = self.i1
+            self.p2 = DummyIterator()
+            self.match = CoGroupIterator.SECOND_EMPTY
+            return True
+        else:
+            if self.key == self.i2.key:
+                self.p1 = self.i1
+                self.p2 = self.i2
+                self.match = CoGroupIterator.NONE_REMAINED
+            elif self.key < self.i2.key:
+                self.p1 = self.i1
+                self.p2 = DummyIterator()
+                self.match = CoGroupIterator.SECOND_REMAINED
+            else:
+                self.p1 = DummyIterator()
+                self.p2 = self.i2
+                self.match = CoGroupIterator.FIRST_REMAINED
+            return True
+
+
+class Iterator(defIter.Iterator):
+    def __init__(self, con, group=0):
+        super(Iterator, self).__init__()
+        self._connection = con
+        self._init = True
+        self._group = group
+        self._deserializer = None
+
+    def __next__(self):
+        return self.next()
+
+    def next(self):
+        if self.has_next():
+            if self._deserializer is None:
+                self._deserializer = _get_deserializer(self._group, self._connection.read)
+            return self._deserializer.deserialize()
+        else:
+            raise StopIteration
+
+    def has_next(self):
+        return self._connection.has_next(self._group)
+
+    def _reset(self):
+        self._deserializer = None
+
+
+class DummyIterator(Iterator):
+    def __init__(self):
+        super(Iterator, self).__init__()
+
+    def __next__(self):
+        raise StopIteration
+
+    def next(self):
+        raise StopIteration
+
+    def has_next(self):
+        return False
+
+
+def _get_deserializer(group, read, type=None):
+    if type is None:
+        type = read(1, group)
+        return _get_deserializer(group, read, type)
+    elif type == Types.TYPE_TUPLE:
+        return TupleDeserializer(read, group)
+    elif type == Types.TYPE_BYTE:
+        return ByteDeserializer(read, group)
+    elif type == Types.TYPE_BYTES:
+        return ByteArrayDeserializer(read, group)
+    elif type == Types.TYPE_BOOLEAN:
+        return BooleanDeserializer(read, group)
+    elif type == Types.TYPE_FLOAT:
+        return FloatDeserializer(read, group)
+    elif type == Types.TYPE_DOUBLE:
+        return DoubleDeserializer(read, group)
+    elif type == Types.TYPE_INTEGER:
+        return IntegerDeserializer(read, group)
+    elif type == Types.TYPE_LONG:
+        return LongDeserializer(read, group)
+    elif type == Types.TYPE_STRING:
+        return StringDeserializer(read, group)
+    elif type == Types.TYPE_NULL:
+        return NullDeserializer(read, group)
+
+
+class TupleDeserializer(object):
+    def __init__(self, read, group):
+        self.read = read
+        self._group = group
+        size = unpack(">I", self.read(4, self._group))[0]
+        self.deserializer = [_get_deserializer(self._group, self.read) for _ in range(size)]
+
+    def deserialize(self):
+        return tuple([s.deserialize() for s in self.deserializer])
+
+
+class ByteDeserializer(object):
+    def __init__(self, read, group):
+        self.read = read
+        self._group = group
+
+    def deserialize(self):
+        return unpack(">c", self.read(1, self._group))[0]
+
+
+class ByteArrayDeserializer(object):
+    def __init__(self, read, group):
+        self.read = read
+        self._group = group
+
+    def deserialize(self):
+        size = unpack(">i", self.read(4, self._group))[0]
+        return bytearray(self.read(size, self._group)) if size else bytearray(b"")
+
+
+class BooleanDeserializer(object):
+    def __init__(self, read, group):
+        self.read = read
+        self._group = group
+
+    def deserialize(self):
+        return unpack(">?", self.read(1, self._group))[0]
+
+
+class FloatDeserializer(object):
+    def __init__(self, read, group):
+        self.read = read
+        self._group = group
+
+    def deserialize(self):
+        return unpack(">f", self.read(4, self._group))[0]
+
+
+class DoubleDeserializer(object):
+    def __init__(self, read, group):
+        self.read = read
+        self._group = group
+
+    def deserialize(self):
+        return unpack(">d", self.read(8, self._group))[0]
+
+
+class IntegerDeserializer(object):
+    def __init__(self, read, group):
+        self.read = read
+        self._group = group
+
+    def deserialize(self):
+        return unpack(">i", self.read(4, self._group))[0]
+
+
+class LongDeserializer(object):
+    def __init__(self, read, group):
+        self.read = read
+        self._group = group
+
+    def deserialize(self):
+        return unpack(">q", self.read(8, self._group))[0]
+
+
+class StringDeserializer(object):
+    def __init__(self, read, group):
+        self.read = read
+        self._group = group
+
+    def deserialize(self):
+        length = unpack(">i", self.read(4, self._group))[0]
+        return self.read(length, self._group).decode("utf-8") if length else ""
+
+
+class NullDeserializer(object):
+    def __init__(self, read, group):
+        self.read = read
+        self._group = group
+
+    def deserialize(self):
+        return None

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/__init__.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/__init__.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/__init__.py
new file mode 100644
index 0000000..65b48d4
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/__init__.py
@@ -0,0 +1,17 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery10.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery10.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery10.py
new file mode 100644
index 0000000..032ef85
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery10.py
@@ -0,0 +1,115 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import sys
+
+from datetime import datetime
+from flink.plan.Environment import get_environment
+from flink.plan.Constants import INT, STRING, FLOAT, WriteMode
+from flink.functions.FilterFunction import FilterFunction
+from flink.functions.ReduceFunction import ReduceFunction
+from flink.functions.JoinFunction import JoinFunction
+from flink.functions.MapFunction import MapFunction
+
+class OrderFilter(FilterFunction):
+    def filter(self, value):
+        if (datetime.strptime(value[2], "%Y-%m-%d") > datetime.strptime("1990-12-31", "%Y-%m-%d")):
+            return True
+        else:
+            return False
+
+class LineitemFilter(FilterFunction):
+    def filter(self, value):
+    	if value[3] == "R":
+    		return True
+    	else:
+    		return False
+
+class ComputeRevenue(MapFunction):
+	def map(self, value):
+		return (value[0], value[1] * (1 - value[2]))
+
+class SumReducer(ReduceFunction):
+    def reduce(self, value1, value2):
+        return (value1[0], value1[1] + value2[1])
+
+class test(JoinFunction):
+	def join(self, value1, value2):
+		return (value1[1], value2[1])
+
+
+if __name__ == "__main__":
+
+	env = get_environment()
+	
+	if len(sys.argv) != 6:
+		sys.exit("Usage: ./bin/pyflink.sh TPCHQuery10 <customer path> <order path> <lineitem path> <nation path> <output path>")
+	
+	customer = env \
+        .read_csv(sys.argv[1], 
+            [INT, STRING, STRING, INT, STRING, FLOAT, STRING, STRING], '\n', '|') \
+        .project(0,1,2,3,5) 
+
+	order = env \
+        .read_csv(sys.argv[2], 
+            [INT, INT, STRING, FLOAT, STRING, STRING, STRING, INT, STRING], '\n', '|') \
+        .project(0,1,4) \
+   		.filter(OrderFilter()) \
+   		.project(0,1)
+
+	lineitem = env \
+        .read_csv(sys.argv[3], [INT, INT, INT, INT, INT, FLOAT, FLOAT, FLOAT,
+             STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING], '\n', '|') \
+        .project(0,5,6,8) \
+        .filter(LineitemFilter()) \
+        .map(ComputeRevenue(), [INT, FLOAT])
+
+	nation = env \
+    	.read_csv(sys.argv[4], [INT, STRING, INT, STRING], '\n', '|') \
+    	.project(0,1)
+
+	revenueByCustomer = order \
+    	.join_with_huge(lineitem) \
+    	.where(0) \
+    	.equal_to(0) \
+    	.project_first(1) \
+    	.project_second(1)
+
+	revenueByCustomer = revenueByCustomer \
+    	.group_by(0) \
+    	.reduce(SumReducer())
+
+	customerWithNation = customer \
+    	.join_with_tiny(nation) \
+    	.where(3) \
+    	.equal_to(0) \
+    	.project_first(0,1,2) \
+    	.project_second(1) \
+    	.project_first(4)
+
+	result = customerWithNation \
+    	.join(revenueByCustomer) \
+    	.where(0) \
+    	.equal_to(0) \
+    	.project_first(0,1,2,3,4) \
+    	.project_second(1)
+
+	result.write_csv(sys.argv[5], '\n', '|', WriteMode.OVERWRITE)
+
+	env.set_degree_of_parallelism(1)
+
+	env.execute(local=True)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery3.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery3.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery3.py
new file mode 100644
index 0000000..5fafb01
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery3.py
@@ -0,0 +1,106 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import sys
+
+from datetime import datetime
+from flink.plan.Environment import get_environment
+from flink.plan.Constants import INT, STRING, FLOAT, WriteMode
+from flink.functions.FilterFunction import FilterFunction
+from flink.functions.ReduceFunction import ReduceFunction
+from flink.functions.JoinFunction import JoinFunction
+
+class CustomerFilter(FilterFunction):
+    def filter(self, value):
+        if value[1] == "AUTOMOBILE":
+            return True
+        else:
+            return False
+
+class LineitemFilter(FilterFunction):
+    def filter(self, value):
+        if (datetime.strptime(value[3], "%Y-%m-%d") > datetime.strptime("1995-03-12", "%Y-%m-%d")):
+            return True
+        else:
+            return False
+
+class OrderFilter(FilterFunction):
+    def filter(self, value):
+        if (datetime.strptime(value[2], "%Y-%m-%d") < datetime.strptime("1995-03-12", "%Y-%m-%d")):
+            return True
+        else:
+            return False
+
+class SumReducer(ReduceFunction):
+    def reduce(self, value1, value2):
+        return (value1[0], value1[1] + value2[1], value1[2], value1[3])
+
+class CustomerOrderJoin(JoinFunction):
+    def join(self, value1, value2):
+        return (value2[0], 0.0, value2[2], value2[3])
+
+class CustomerOrderLineitemJoin(JoinFunction):
+    def join(self, value1, value2):
+        return (value1[0], value2[1] * (1 - value2[2]), value1[2], value1[3])
+
+
+
+if __name__ == "__main__":
+    env = get_environment()
+
+    if len(sys.argv) != 5:
+        sys.exit("Usage: ./bin/pyflink.sh TPCHQuery3 <lineitem path> <customer path> <order path> <output path>")
+    lineitem = env \
+        .read_csv(sys.argv[1], [INT, INT, INT, INT, INT, FLOAT, FLOAT, FLOAT,
+             STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING], '\n', '|') \
+        .project(0,5,6,10) \
+        .filter(LineitemFilter())
+
+
+    customer = env \
+        .read_csv(sys.argv[2], 
+            [INT, STRING, STRING, INT, STRING, FLOAT, STRING, STRING], '\n', '|') \
+        .project(0,6) \
+        .filter(CustomerFilter())
+
+    order = env \
+        .read_csv(sys.argv[3], 
+            [INT, INT, STRING, FLOAT, STRING, STRING, STRING, INT, STRING], '\n', '|') \
+        .project(0,1,4,7) \
+        .filter(OrderFilter())
+
+    customerWithOrder = customer \
+        .join(order) \
+        .where(0) \
+        .equal_to(1) \
+        .using(CustomerOrderJoin(),[INT, FLOAT, STRING, INT])
+
+    result = customerWithOrder \
+        .join(lineitem) \
+        .where(0) \
+        .equal_to(0) \
+        .using(CustomerOrderLineitemJoin(), [INT, FLOAT, STRING, INT]) \
+        .group_by(0, 2, 3) \
+        .reduce(SumReducer())
+
+    result.write_csv(sys.argv[4], '\n', '|', WriteMode.OVERWRITE)
+
+    env.set_degree_of_parallelism(1)
+
+    env.execute(local=True)
+
+

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TriangleEnumeration.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TriangleEnumeration.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TriangleEnumeration.py
new file mode 100644
index 0000000..2727635
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TriangleEnumeration.py
@@ -0,0 +1,152 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.plan.Environment import get_environment
+from flink.plan.Constants import INT, Order
+from flink.functions.FlatMapFunction import FlatMapFunction
+from flink.functions.GroupReduceFunction import GroupReduceFunction
+from flink.functions.ReduceFunction import ReduceFunction
+from flink.functions.MapFunction import MapFunction
+from flink.functions.JoinFunction import JoinFunction
+
+
+class EdgeDuplicator(FlatMapFunction):
+    def flat_map(self, value, collector):
+        collector.collect((value[0], value[1]))
+        collector.collect((value[1], value[0]))
+
+
+class DegreeCounter(GroupReduceFunction):
+    def reduce(self, iterator, collector):
+        other_vertices = []
+
+        data = iterator.next()
+        edge = (data[0], data[1])
+
+        group_vertex = edge[0]
+        other_vertices.append(edge[1])
+
+        while iterator.has_next():
+            data = iterator.next()
+            edge = [data[0], data[1]]
+            other_vertex = edge[1]
+
+            contained = False
+            for v in other_vertices:
+                if v == other_vertex:
+                    contained = True
+                    break
+            if not contained and not other_vertex == group_vertex:
+                other_vertices.append(other_vertex)
+
+        degree = len(other_vertices)
+
+        for other_vertex in other_vertices:
+            if group_vertex < other_vertex:
+                output_edge = (group_vertex, degree, other_vertex, 0)
+            else:
+                output_edge = (other_vertex, 0, group_vertex, degree)
+            collector.collect(output_edge)
+
+
+class DegreeJoiner(ReduceFunction):
+    def reduce(self, value1, value2):
+        edge1 = [value1[0], value1[1], value1[2], value1[3]]
+        edge2 = [value2[0], value2[1], value2[2], value2[3]]
+
+        out_edge = [edge1[0], edge1[1], edge1[2], edge1[3]]
+        if edge1[1] == 0 and (not edge1[3] == 0):
+            out_edge[1] = edge2[1]
+        elif (not edge1[1] == 0) and edge1[3] == 0:
+            out_edge[3] = edge2[3]
+        return out_edge
+
+
+class EdgeByDegreeProjector(MapFunction):
+    def map(self, value):
+        if value[1] > value[3]:
+            return (value[2], value[0])
+        else:
+            return (value[0], value[2])
+
+
+class EdgeByIdProjector(MapFunction):
+    def map(self, value):
+        edge = (value[0], value[1])
+        if value[0] > value[1]:
+            return (value[1], value[0])
+        else:
+            return (value[0], value[1])
+
+
+class TriadBuilder(GroupReduceFunction):
+    def reduce(self, iterator, collector):
+        vertices = []
+
+        y = iterator.next()
+        first_edge = [y[0], y[1]]
+
+        vertices.append(first_edge[1])
+
+        while iterator.has_next():
+            x = iterator.next()
+            second_edge = [x[0], x[1]]
+            higher_vertex_id = second_edge[1]
+
+            for lowerVertexId in vertices:
+                collector.collect((first_edge[0], lowerVertexId, higher_vertex_id))
+            vertices.append(higher_vertex_id)
+
+
+class TriadFilter(JoinFunction):
+    def join(self, value1, value2):
+        return value1
+
+
+if __name__ == "__main__":
+    env = get_environment()
+    edges = env.from_elements(
+        (1, 2), (1, 3), (1, 4), (1, 5), (2, 3), (2, 5), (3, 4), (3, 7), (3, 8), (5, 6), (7, 8))
+
+    edges_with_degrees = edges \
+        .flat_map(EdgeDuplicator(), [INT, INT]) \
+        .group_by(0) \
+        .sort_group(1, Order.ASCENDING) \
+        .reduce_group(DegreeCounter(), [INT, INT, INT, INT]) \
+        .group_by(0, 2) \
+        .reduce(DegreeJoiner())
+
+    edges_by_degree = edges_with_degrees \
+        .map(EdgeByDegreeProjector(), [INT, INT])
+
+    edges_by_id = edges_by_degree \
+        .map(EdgeByIdProjector(), [INT, INT])
+
+    triangles = edges_by_degree \
+        .group_by(0) \
+        .sort_group(1, Order.ASCENDING) \
+        .reduce_group(TriadBuilder(), [INT, INT, INT]) \
+        .join(edges_by_id) \
+        .where(1, 2) \
+        .equal_to(0, 1) \
+        .using(TriadFilter(), [INT, INT, INT])
+
+    triangles.output()
+
+    env.set_degree_of_parallelism(1)
+
+    env.execute(local=True)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WebLogAnalysis.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WebLogAnalysis.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WebLogAnalysis.py
new file mode 100644
index 0000000..d571cf9
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WebLogAnalysis.py
@@ -0,0 +1,87 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import sys
+
+from datetime import datetime
+from flink.plan.Environment import get_environment
+from flink.plan.Constants import INT, STRING, FLOAT, WriteMode
+from flink.functions.CoGroupFunction import CoGroupFunction
+from flink.functions.FilterFunction import FilterFunction
+
+
+class DocumentFilter(FilterFunction):
+    def filter(self, value):
+        keywords = [" editors ", " oscillations "]
+        for kw in keywords:
+            if kw not in value[1]:
+                return False
+        return True
+
+class RankFilter(FilterFunction):
+    def filter(self, value):
+        return value[0] > 40
+
+class VisitFilter(FilterFunction):
+    def filter(self, value):
+        return datetime.strptime(value[1], "%Y-%m-%d").year == 2007
+
+class AntiJoinVisits(CoGroupFunction):
+    def co_group(self, iterator1, iterator2, collector):
+        if not iterator2.has_next():
+            while iterator1.has_next():
+                collector.collect(iterator1.next())
+
+if __name__ == "__main__":
+
+    env = get_environment()
+
+    if len(sys.argv) != 5:
+    	sys.exit("Usage: ./bin/pyflink.sh WebLogAnalysis <docments path> <ranks path> <visits path> <output path>")
+
+    documents = env \
+        .read_csv(sys.argv[1], [STRING, STRING], "\n", "|") \
+        .filter(DocumentFilter()) \
+        .project(0)
+
+    ranks = env \
+        .read_csv(sys.argv[2], [INT, STRING, INT], "\n", "|") \
+        .filter(RankFilter())
+
+    visits = env \
+        .read_csv(sys.argv[3], [STRING, STRING, STRING, FLOAT, STRING, STRING, STRING, STRING, INT], "\n", "|") \
+        .project(1,2) \
+        .filter(VisitFilter()) \
+        .project(0)
+
+    docWithRanks = documents \
+        .join(ranks) \
+        .where(0) \
+        .equal_to(1) \
+        .project_second(0,1,2)
+
+    result = docWithRanks \
+        .co_group(visits) \
+        .where(1) \
+        .equal_to(0) \
+        .using(AntiJoinVisits(), [INT, STRING, INT])
+
+    result.write_csv(sys.argv[4], '\n', '|', WriteMode.OVERWRITE)
+
+    env.set_degree_of_parallelism(1)
+
+    env.execute(local=True)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WordCount.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WordCount.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WordCount.py
new file mode 100644
index 0000000..8a89a6f
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WordCount.py
@@ -0,0 +1,61 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import sys
+
+from flink.plan.Environment import get_environment
+from flink.plan.Constants import INT, STRING
+from flink.functions.FlatMapFunction import FlatMapFunction
+from flink.functions.GroupReduceFunction import GroupReduceFunction
+
+
+class Tokenizer(FlatMapFunction):
+    def flat_map(self, value, collector):
+        for word in value.lower().split():
+            collector.collect((1, word))
+
+
+class Adder(GroupReduceFunction):
+    def reduce(self, iterator, collector):
+        count, word = iterator.next()
+        count += sum([x[0] for x in iterator])
+        collector.collect((count, word))
+
+
+if __name__ == "__main__":
+    env = get_environment()
+    if len(sys.argv) != 1 and len(sys.argv) != 3:
+        sys.exit("Usage: ./bin/pyflink.sh WordCount[ - <text path> <result path>]")
+
+    if len(sys.argv) == 3:
+        data = env.read_text(sys.argv[1])
+    else:
+        data = env.from_elements("hello","world","hello","car","tree","data","hello")
+
+    result = data \
+        .flat_map(Tokenizer(), (INT, STRING)) \
+        .group_by(1) \
+        .reduce_group(Adder(), (INT, STRING), combinable=True) \
+
+    if len(sys.argv) == 3:
+        result.write_csv(sys.argv[2])
+    else:
+        result.output()
+
+    env.set_degree_of_parallelism(1)
+
+    env.execute(local=True)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/__init__.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/__init__.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/__init__.py
new file mode 100644
index 0000000..d35bf39
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/__init__.py
@@ -0,0 +1,17 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CoGroupFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CoGroupFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CoGroupFunction.py
new file mode 100644
index 0000000..db951fe
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CoGroupFunction.py
@@ -0,0 +1,53 @@
+# ###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.functions import Function, RuntimeContext
+from flink.connection import Iterator, Connection, Collector
+
+
+class CoGroupFunction(Function.Function):
+    def __init__(self):
+        super(CoGroupFunction, self).__init__()
+        self._keys1 = None
+        self._keys2 = None
+
+    def _configure(self, input_file, output_file, port):
+        self._connection = Connection.TwinBufferingTCPMappedFileConnection(input_file, output_file, port)
+        self._iterator = Iterator.Iterator(self._connection, 0)
+        self._iterator2 = Iterator.Iterator(self._connection, 1)
+        self._cgiter = Iterator.CoGroupIterator(self._iterator, self._iterator2, self._keys1, self._keys2)
+        self.context = RuntimeContext.RuntimeContext(self._iterator, self._collector)
+        self._configure_chain(Collector.Collector(self._connection))
+
+    def _run(self):
+        collector = self._collector
+        iterator = self._cgiter
+        function = self.co_group
+        iterator._init()
+        while iterator.next():
+            result = function(iterator.p1, iterator.p2, collector)
+            if result is not None:
+                for res in result:
+                    collector.collect(res)
+            while iterator.p1.has_next():
+                iterator.p1.next()
+            while iterator.p2.has_next():
+                iterator.p2.next()
+        collector._close()
+
+    def co_group(self, iterator1, iterator2, collector):
+        pass
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CrossFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CrossFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CrossFunction.py
new file mode 100644
index 0000000..6657d45
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CrossFunction.py
@@ -0,0 +1,34 @@
+# ###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.functions import Function
+
+
+class CrossFunction(Function.Function):
+    def __init__(self):
+        super(CrossFunction, self).__init__()
+
+    def _run(self):
+        collector = self._collector
+        function = self.cross
+        iterator = self._iterator
+        for value in iterator:
+            collector.collect(function(value[0], value[1]))
+        collector._close()
+
+    def cross(self, value1, value2):
+        pass
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FilterFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FilterFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FilterFunction.py
new file mode 100644
index 0000000..fb4fb74
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FilterFunction.py
@@ -0,0 +1,38 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.functions import Function
+
+
+class FilterFunction(Function.Function):
+    def __init__(self):
+        super(FilterFunction, self).__init__()
+
+    def _run(self):
+        collector = self._collector
+        function = self.filter
+        for value in self._iterator:
+            if function(value):
+                collector.collect(value)
+        collector._close()
+
+    def collect(self, value):
+        if self.filter(value):
+            self._collector.collect(value)
+
+    def filter(self, value):
+        pass

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FlatMapFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FlatMapFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FlatMapFunction.py
new file mode 100644
index 0000000..ebc493c
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FlatMapFunction.py
@@ -0,0 +1,44 @@
+# ###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.functions import Function
+
+
+class FlatMapFunction(Function.Function):
+    def __init__(self):
+        super(FlatMapFunction, self).__init__()
+
+    def _run(self):
+        collector = self._collector
+        function = self.flat_map
+        iterator = self._iterator
+        for value in iterator:
+            result = function(value, collector)
+            if result is not None:
+                for res in result:
+                    collector.collect(res)
+        collector._close()
+
+    def collect(self, value):
+        collector = self._collector
+        result = self.flat_map(value, collector)
+        if result is not None:
+            for res in result:
+                collector.collect(res)
+
+    def flat_map(self, value, collector):
+        pass
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/Function.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/Function.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/Function.py
new file mode 100644
index 0000000..5323462
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/Function.py
@@ -0,0 +1,83 @@
+# ###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from abc import ABCMeta, abstractmethod
+import sys
+from collections import deque
+from flink.connection import Connection, Iterator, Collector
+from flink.functions import RuntimeContext
+
+
+class Function(object):
+    __metaclass__ = ABCMeta
+
+    def __init__(self):
+        self._connection = None
+        self._iterator = None
+        self._collector = None
+        self.context = None
+        self._chain_operator = None
+
+    def _configure(self, input_file, output_file, port):
+        self._connection = Connection.BufferingTCPMappedFileConnection(input_file, output_file, port)
+        self._iterator = Iterator.Iterator(self._connection)
+        self.context = RuntimeContext.RuntimeContext(self._iterator, self._collector)
+        self._configure_chain(Collector.Collector(self._connection))
+
+    def _configure_chain(self, collector):
+        if self._chain_operator is not None:
+            self._collector = self._chain_operator
+            self._collector.context = self.context
+            self._collector._configure_chain(collector)
+            self._collector._open()
+        else:
+            self._collector = collector
+
+    def _chain(self, operator):
+        self._chain_operator = operator
+
+    @abstractmethod
+    def _run(self):
+        pass
+
+    def _open(self):
+        pass
+
+    def _close(self):
+        self._collector._close()
+
+    def _go(self):
+        self._receive_broadcast_variables()
+        self._run()
+
+    def _receive_broadcast_variables(self):
+        broadcast_count = self._iterator.next()
+        self._iterator._reset()
+        self._connection.reset()
+        for _ in range(broadcast_count):
+            name = self._iterator.next()
+            self._iterator._reset()
+            self._connection.reset()
+            bc = deque()
+            while(self._iterator.has_next()):
+                bc.append(self._iterator.next())
+            self.context._add_broadcast_variable(name, bc)
+            self._iterator._reset()
+            self._connection.reset()
+
+
+

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/GroupReduceFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/GroupReduceFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/GroupReduceFunction.py
new file mode 100644
index 0000000..11bba30
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/GroupReduceFunction.py
@@ -0,0 +1,127 @@
+# ###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from collections import defaultdict
+from flink.functions import Function, RuntimeContext
+from flink.connection import Connection, Iterator, Collector
+from flink.plan.Constants import Order
+
+
+class GroupReduceFunction(Function.Function):
+    def __init__(self):
+        super(GroupReduceFunction, self).__init__()
+        self._keys = None
+        self._sort_ops = []
+        self._combine = False
+        self._values = []
+
+    def _configure(self, input_file, output_file, port):
+        if self._combine:
+            self._connection = Connection.BufferingTCPMappedFileConnection(input_file, output_file, port)
+            self._iterator = Iterator.Iterator(self._connection)
+            self._collector = Collector.Collector(self._connection)
+            self.context = RuntimeContext.RuntimeContext(self._iterator, self._collector)
+            self._run = self._run_combine
+        else:
+            self._connection = Connection.BufferingTCPMappedFileConnection(input_file, output_file, port)
+            self._iterator = Iterator.Iterator(self._connection)
+            self._group_iterator = Iterator.GroupIterator(self._iterator, self._keys)
+            self.context = RuntimeContext.RuntimeContext(self._iterator, self._collector)
+            self._configure_chain(Collector.Collector(self._connection))
+        self._open()
+
+    def _open(self):
+        if self._keys is None:
+            self._extract_keys = self._extract_keys_id
+
+    def _close(self):
+        self._sort_and_combine()
+        self._collector._close()
+
+    def _set_grouping_keys(self, keys):
+        self._keys = keys
+
+    def _set_sort_ops(self, ops):
+        self._sort_ops = ops
+
+    def _run(self):#reduce
+        connection = self._connection
+        collector = self._collector
+        function = self.reduce
+        iterator = self._group_iterator
+        iterator._init()
+        while iterator.has_group():
+            iterator.next_group()
+            result = function(iterator, collector)
+            if result is not None:
+                for value in result:
+                    collector.collect(value)
+        collector._close()
+        connection.send_end_signal()
+
+    def _run_combine(self):#unchained combine
+        connection = self._connection
+        collector = self._collector
+        function = self.combine
+        iterator = self._iterator
+        while 1:
+            result = function(iterator, collector)
+            if result is not None:
+                for value in result:
+                    collector.collect(value)
+            connection.send_end_signal()
+            connection.reset()
+
+    def collect(self, value):#chained combine
+        self._values.append(value)
+        if len(self._values) > 1000:
+            self._sort_and_combine()
+
+    def _sort_and_combine(self):
+        values = self._values
+        function = self.combine
+        collector = self._collector
+        extractor = self._extract_keys
+        grouping = defaultdict(list)
+        for value in values:
+            grouping[extractor(value)].append(value)
+        keys = list(grouping.keys())
+        keys.sort()
+        for key in keys:
+            values = grouping[key]
+            for op in reversed(self._sort_ops):
+                values.sort(key=lambda x:x[op[0]], reverse = op[1] == Order.DESCENDING)
+            result = function(Iterator.ListIterator(values), collector)
+            if result is not None:
+                for res in result:
+                    collector.collect(res)
+        self._values = []
+
+    def _extract_keys(self, x):
+        return tuple([x[k] for k in self._keys])
+
+    def _extract_sort_keys(self, x):
+        return tuple(x[k] for k in self._sort_keys)
+
+    def _extract_keys_id(self, x):
+        return x
+
+    def reduce(self, iterator, collector):
+        pass
+
+    def combine(self, iterator, collector):
+        self.reduce(iterator, collector)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/JoinFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/JoinFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/JoinFunction.py
new file mode 100644
index 0000000..0c1274b
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/JoinFunction.py
@@ -0,0 +1,33 @@
+# ###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.functions import Function
+
+
+class JoinFunction(Function.Function):
+    def __init__(self):
+        super(JoinFunction, self).__init__()
+
+    def _run(self):
+        collector = self._collector
+        function = self.join
+        for value in self._iterator:
+            collector.collect(function(value[0], value[1]))
+        collector._close()
+
+    def join(self, value1, value2):
+        pass