You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2015/11/12 21:09:22 UTC
[7/8] flink git commit: [FLINK-2901] Move PythonAPI to flink-libraries
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/Sender.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/Sender.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/Sender.java
new file mode 100644
index 0000000..b897f12
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/Sender.java
@@ -0,0 +1,411 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.flink.python.api.streaming;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.java.tuple.Tuple;
+import static org.apache.flink.python.api.PythonPlanBinder.FLINK_TMP_DATA_DIR;
+import static org.apache.flink.python.api.PythonPlanBinder.MAPPED_FILE_SIZE;
+
+/**
+ * General-purpose class to write data to memory-mapped files.
+ */
+public class Sender implements Serializable {
+ public static final byte TYPE_TUPLE = (byte) 11;
+ public static final byte TYPE_BOOLEAN = (byte) 10;
+ public static final byte TYPE_BYTE = (byte) 9;
+ public static final byte TYPE_SHORT = (byte) 8;
+ public static final byte TYPE_INTEGER = (byte) 7;
+ public static final byte TYPE_LONG = (byte) 6;
+ public static final byte TYPE_DOUBLE = (byte) 4;
+ public static final byte TYPE_FLOAT = (byte) 5;
+ public static final byte TYPE_CHAR = (byte) 3;
+ public static final byte TYPE_STRING = (byte) 2;
+ public static final byte TYPE_BYTES = (byte) 1;
+ public static final byte TYPE_NULL = (byte) 0;
+
+ private final AbstractRichFunction function;
+
+ private File outputFile;
+ private RandomAccessFile outputRAF;
+ private FileChannel outputChannel;
+ private MappedByteBuffer fileBuffer;
+
+ private final ByteBuffer[] saved = new ByteBuffer[2];
+
+ private final Serializer[] serializer = new Serializer[2];
+
+ public Sender(AbstractRichFunction function) {
+ this.function = function;
+ }
+
+ //=====Setup========================================================================================================
+ public void open(String path) throws IOException {
+ setupMappedFile(path);
+ }
+
+ private void setupMappedFile(String outputFilePath) throws FileNotFoundException, IOException {
+ File x = new File(FLINK_TMP_DATA_DIR);
+ x.mkdirs();
+
+ outputFile = new File(outputFilePath);
+ if (outputFile.exists()) {
+ outputFile.delete();
+ }
+ outputFile.createNewFile();
+ outputRAF = new RandomAccessFile(outputFilePath, "rw");
+ outputRAF.setLength(MAPPED_FILE_SIZE);
+ outputRAF.seek(MAPPED_FILE_SIZE - 1);
+ outputRAF.writeByte(0);
+ outputRAF.seek(0);
+ outputChannel = outputRAF.getChannel();
+ fileBuffer = outputChannel.map(FileChannel.MapMode.READ_WRITE, 0, MAPPED_FILE_SIZE);
+ }
+
+ public void close() throws IOException {
+ closeMappedFile();
+ }
+
+ private void closeMappedFile() throws IOException {
+ outputChannel.close();
+ outputRAF.close();
+ }
+
+ /**
+ * Resets this object to the post-configuration state.
+ */
+ public void reset() {
+ serializer[0] = null;
+ serializer[1] = null;
+ fileBuffer.clear();
+ }
+
+ //=====Serialization================================================================================================
+ /**
+ * Writes a single record to the memory-mapped file. This method does NOT take care of synchronization. The user
+ * must guarantee that the file may be written to before calling this method. This method essentially reserves the
+ * whole buffer for one record. As such it imposes some performance restrictions and should only be used when
+ * absolutely necessary.
+ *
+ * @param value record to send
+ * @return size of the written buffer
+ * @throws IOException
+ */
+ public int sendRecord(Object value) throws IOException {
+ fileBuffer.clear();
+ int group = 0;
+
+ serializer[group] = getSerializer(value);
+ ByteBuffer bb = serializer[group].serialize(value);
+ if (bb.remaining() > MAPPED_FILE_SIZE) {
+ throw new RuntimeException("Serialized object does not fit into a single buffer.");
+ }
+ fileBuffer.put(bb);
+
+ int size = fileBuffer.position();
+
+ reset();
+ return size;
+ }
+
+ public boolean hasRemaining(int group) {
+ return saved[group] != null;
+ }
+
+ /**
+ * Extracts records from an iterator and writes them to the memory-mapped file. This method assumes that all values
+ * in the iterator are of the same type. This method does NOT take care of synchronization. The caller must
+ * guarantee that the file may be written to before calling this method.
+ *
+ * @param i iterator containing records
+ * @param group group to which the iterator belongs, most notably used by CoGroup-functions.
+ * @return size of the written buffer
+ * @throws IOException
+ */
+ public int sendBuffer(Iterator i, int group) throws IOException {
+ fileBuffer.clear();
+
+ Object value;
+ ByteBuffer bb;
+ if (serializer[group] == null) {
+ value = i.next();
+ serializer[group] = getSerializer(value);
+ bb = serializer[group].serialize(value);
+ if (bb.remaining() > MAPPED_FILE_SIZE) {
+ throw new RuntimeException("Serialized object does not fit into a single buffer.");
+ }
+ fileBuffer.put(bb);
+
+ }
+ if (saved[group] != null) {
+ fileBuffer.put(saved[group]);
+ saved[group] = null;
+ }
+ while (i.hasNext() && saved[group] == null) {
+ value = i.next();
+ bb = serializer[group].serialize(value);
+ if (bb.remaining() > MAPPED_FILE_SIZE) {
+ throw new RuntimeException("Serialized object does not fit into a single buffer.");
+ }
+ if (bb.remaining() <= fileBuffer.remaining()) {
+ fileBuffer.put(bb);
+ } else {
+ saved[group] = bb;
+ }
+ }
+
+ int size = fileBuffer.position();
+ return size;
+ }
+
+ private enum SupportedTypes {
+ TUPLE, BOOLEAN, BYTE, BYTES, CHARACTER, SHORT, INTEGER, LONG, FLOAT, DOUBLE, STRING, OTHER, NULL
+ }
+
+ //=====Serializer===================================================================================================
+ private Serializer getSerializer(Object value) throws IOException {
+ String className = value.getClass().getSimpleName().toUpperCase();
+ if (className.startsWith("TUPLE")) {
+ className = "TUPLE";
+ }
+ if (className.startsWith("BYTE[]")) {
+ className = "BYTES";
+ }
+ SupportedTypes type = SupportedTypes.valueOf(className);
+ switch (type) {
+ case TUPLE:
+ fileBuffer.put(TYPE_TUPLE);
+ fileBuffer.putInt(((Tuple) value).getArity());
+ return new TupleSerializer((Tuple) value);
+ case BOOLEAN:
+ fileBuffer.put(TYPE_BOOLEAN);
+ return new BooleanSerializer();
+ case BYTE:
+ fileBuffer.put(TYPE_BYTE);
+ return new ByteSerializer();
+ case BYTES:
+ fileBuffer.put(TYPE_BYTES);
+ return new BytesSerializer();
+ case CHARACTER:
+ fileBuffer.put(TYPE_CHAR);
+ return new CharSerializer();
+ case SHORT:
+ fileBuffer.put(TYPE_SHORT);
+ return new ShortSerializer();
+ case INTEGER:
+ fileBuffer.put(TYPE_INTEGER);
+ return new IntSerializer();
+ case LONG:
+ fileBuffer.put(TYPE_LONG);
+ return new LongSerializer();
+ case STRING:
+ fileBuffer.put(TYPE_STRING);
+ return new StringSerializer();
+ case FLOAT:
+ fileBuffer.put(TYPE_FLOAT);
+ return new FloatSerializer();
+ case DOUBLE:
+ fileBuffer.put(TYPE_DOUBLE);
+ return new DoubleSerializer();
+ case NULL:
+ fileBuffer.put(TYPE_NULL);
+ return new NullSerializer();
+ default:
+ throw new IllegalArgumentException("Unknown Type encountered: " + type);
+ }
+ }
+
+ private abstract class Serializer<T> {
+ protected ByteBuffer buffer;
+
+ public Serializer(int capacity) {
+ buffer = ByteBuffer.allocate(capacity);
+ }
+
+ public ByteBuffer serialize(T value) {
+ buffer.clear();
+ serializeInternal(value);
+ buffer.flip();
+ return buffer;
+ }
+
+ public abstract void serializeInternal(T value);
+ }
+
+ private class ByteSerializer extends Serializer<Byte> {
+ public ByteSerializer() {
+ super(1);
+ }
+
+ @Override
+ public void serializeInternal(Byte value) {
+ buffer.put(value);
+ }
+ }
+
+ private class BooleanSerializer extends Serializer<Boolean> {
+ public BooleanSerializer() {
+ super(1);
+ }
+
+ @Override
+ public void serializeInternal(Boolean value) {
+ buffer.put(value ? (byte) 1 : (byte) 0);
+ }
+ }
+
+ private class CharSerializer extends Serializer<Character> {
+ public CharSerializer() {
+ super(4);
+ }
+
+ @Override
+ public void serializeInternal(Character value) {
+ buffer.put((value + "").getBytes());
+ }
+ }
+
+ private class ShortSerializer extends Serializer<Short> {
+ public ShortSerializer() {
+ super(2);
+ }
+
+ @Override
+ public void serializeInternal(Short value) {
+ buffer.putShort(value);
+ }
+ }
+
+ private class IntSerializer extends Serializer<Integer> {
+ public IntSerializer() {
+ super(4);
+ }
+
+ @Override
+ public void serializeInternal(Integer value) {
+ buffer.putInt(value);
+ }
+ }
+
+ private class LongSerializer extends Serializer<Long> {
+ public LongSerializer() {
+ super(8);
+ }
+
+ @Override
+ public void serializeInternal(Long value) {
+ buffer.putLong(value);
+ }
+ }
+
+ private class StringSerializer extends Serializer<String> {
+ public StringSerializer() {
+ super(0);
+ }
+
+ @Override
+ public void serializeInternal(String value) {
+ byte[] bytes = value.getBytes();
+ buffer = ByteBuffer.allocate(bytes.length + 4);
+ buffer.putInt(bytes.length);
+ buffer.put(bytes);
+ }
+ }
+
+ private class FloatSerializer extends Serializer<Float> {
+ public FloatSerializer() {
+ super(4);
+ }
+
+ @Override
+ public void serializeInternal(Float value) {
+ buffer.putFloat(value);
+ }
+ }
+
+ private class DoubleSerializer extends Serializer<Double> {
+ public DoubleSerializer() {
+ super(8);
+ }
+
+ @Override
+ public void serializeInternal(Double value) {
+ buffer.putDouble(value);
+ }
+ }
+
+ private class NullSerializer extends Serializer<Object> {
+ public NullSerializer() {
+ super(0);
+ }
+
+ @Override
+ public void serializeInternal(Object value) {
+ }
+ }
+
+ private class BytesSerializer extends Serializer<byte[]> {
+ public BytesSerializer() {
+ super(0);
+ }
+
+ @Override
+ public void serializeInternal(byte[] value) {
+ buffer = ByteBuffer.allocate(4 + value.length);
+ buffer.putInt(value.length);
+ buffer.put(value);
+ }
+ }
+
+ private class TupleSerializer extends Serializer<Tuple> {
+ private final Serializer[] serializer;
+ private final List<ByteBuffer> buffers;
+
+ public TupleSerializer(Tuple value) throws IOException {
+ super(0);
+ serializer = new Serializer[value.getArity()];
+ buffers = new ArrayList();
+ for (int x = 0; x < serializer.length; x++) {
+ serializer[x] = getSerializer(value.getField(x));
+ }
+ }
+
+ @Override
+ public void serializeInternal(Tuple value) {
+ int length = 0;
+ for (int x = 0; x < serializer.length; x++) {
+ serializer[x].buffer.clear();
+ serializer[x].serializeInternal(value.getField(x));
+ length += serializer[x].buffer.position();
+ buffers.add(serializer[x].buffer);
+ }
+ buffer = ByteBuffer.allocate(length);
+ for (ByteBuffer b : buffers) {
+ b.flip();
+ buffer.put(b);
+ }
+ buffers.clear();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/StreamPrinter.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/StreamPrinter.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/StreamPrinter.java
new file mode 100644
index 0000000..e42364b
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/StreamPrinter.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.flink.python.api.streaming;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * Simple utility class to print all contents of an inputstream to stdout.
+ */
+public class StreamPrinter extends Thread {
+ private final BufferedReader reader;
+ private final boolean wrapInException;
+ private StringBuilder msg;
+
+ public StreamPrinter(InputStream stream) {
+ this(stream, false, null);
+ }
+
+ public StreamPrinter(InputStream stream, boolean wrapInException, StringBuilder msg) {
+ this.reader = new BufferedReader(new InputStreamReader(stream));
+ this.wrapInException = wrapInException;
+ this.msg = msg;
+ }
+
+ @Override
+ public void run() {
+ String line;
+ try {
+ if (wrapInException) {
+ while ((line = reader.readLine()) != null) {
+ msg.append("\n" + line);
+ }
+ } else {
+ while ((line = reader.readLine()) != null) {
+ System.out.println(line);
+ System.out.flush();
+ }
+ }
+ } catch (IOException ex) {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/__init__.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/__init__.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/__init__.py
new file mode 100644
index 0000000..d35bf39
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/__init__.py
@@ -0,0 +1,17 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/__init__.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/__init__.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/__init__.py
new file mode 100644
index 0000000..d35bf39
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/__init__.py
@@ -0,0 +1,17 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Collector.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Collector.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Collector.py
new file mode 100644
index 0000000..bf35756
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Collector.py
@@ -0,0 +1,157 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from struct import pack
+import sys
+
+from flink.connection.Constants import Types
+
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+if PY2:
+ stringtype = basestring
+else:
+ stringtype = str
+
+
+class Collector(object):
+ def __init__(self, con):
+ self._connection = con
+ self._serializer = None
+
+ def _close(self):
+ self._connection.send_end_signal()
+
+ def collect(self, value):
+ self._serializer = _get_serializer(self._connection.write, value)
+ self.collect = self._collect
+ self.collect(value)
+
+ def _collect(self, value):
+ self._connection.write(self._serializer.serialize(value))
+
+
+def _get_serializer(write, value):
+ if isinstance(value, (list, tuple)):
+ write(Types.TYPE_TUPLE)
+ write(pack(">I", len(value)))
+ return TupleSerializer(write, value)
+ elif value is None:
+ write(Types.TYPE_NULL)
+ return NullSerializer()
+ elif isinstance(value, stringtype):
+ write(Types.TYPE_STRING)
+ return StringSerializer()
+ elif isinstance(value, bool):
+ write(Types.TYPE_BOOLEAN)
+ return BooleanSerializer()
+ elif isinstance(value, int) or PY2 and isinstance(value, long):
+ write(Types.TYPE_LONG)
+ return LongSerializer()
+ elif isinstance(value, bytearray):
+ write(Types.TYPE_BYTES)
+ return ByteArraySerializer()
+ elif isinstance(value, float):
+ write(Types.TYPE_DOUBLE)
+ return FloatSerializer()
+ else:
+ raise Exception("Unsupported Type encountered.")
+
+
+class TupleSerializer(object):
+ def __init__(self, write, value):
+ self.serializer = [_get_serializer(write, field) for field in value]
+
+ def serialize(self, value):
+ bits = []
+ for i in range(len(value)):
+ bits.append(self.serializer[i].serialize(value[i]))
+ return b"".join(bits)
+
+
+class BooleanSerializer(object):
+ def serialize(self, value):
+ return pack(">?", value)
+
+
+class FloatSerializer(object):
+ def serialize(self, value):
+ return pack(">d", value)
+
+
+class LongSerializer(object):
+ def serialize(self, value):
+ return pack(">q", value)
+
+
+class ByteArraySerializer(object):
+ def serialize(self, value):
+ value = bytes(value)
+ return pack(">I", len(value)) + value
+
+
+class StringSerializer(object):
+ def serialize(self, value):
+ value = value.encode("utf-8")
+ return pack(">I", len(value)) + value
+
+
+class NullSerializer(object):
+ def serialize(self, value):
+ return b""
+
+
+class TypedCollector(object):
+ def __init__(self, con):
+ self._connection = con
+
+ def collect(self, value):
+ if not isinstance(value, (list, tuple)):
+ self._send_field(value)
+ else:
+ self._connection.write(Types.TYPE_TUPLE)
+ meta = pack(">I", len(value))
+ self._connection.write(bytes([meta[3]]) if PY3 else meta[3])
+ for field in value:
+ self.collect(field)
+
+ def _send_field(self, value):
+ if value is None:
+ self._connection.write(Types.TYPE_NULL)
+ elif isinstance(value, stringtype):
+ value = value.encode("utf-8")
+ size = pack(">I", len(value))
+ self._connection.write(b"".join([Types.TYPE_STRING, size, value]))
+ elif isinstance(value, bytes):
+ size = pack(">I", len(value))
+ self._connection.write(b"".join([Types.TYPE_BYTES, size, value]))
+ elif isinstance(value, bool):
+ data = pack(">?", value)
+ self._connection.write(b"".join([Types.TYPE_BOOLEAN, data]))
+ elif isinstance(value, int) or PY2 and isinstance(value, long):
+ data = pack(">q", value)
+ self._connection.write(b"".join([Types.TYPE_LONG, data]))
+ elif isinstance(value, float):
+ data = pack(">d", value)
+ self._connection.write(b"".join([Types.TYPE_DOUBLE, data]))
+ elif isinstance(value, bytearray):
+ value = bytes(value)
+ size = pack(">I", len(value))
+ self._connection.write(b"".join([Types.TYPE_BYTES, size, value]))
+ else:
+ raise Exception("Unsupported Type encountered.")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Connection.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Connection.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Connection.py
new file mode 100644
index 0000000..988bf25
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Connection.py
@@ -0,0 +1,166 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import mmap
+import socket as SOCKET
+from struct import pack, unpack
+from collections import deque
+import sys
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+MAPPED_FILE_SIZE = 1024 * 1024 * 64
+
+SIGNAL_REQUEST_BUFFER = b"\x00\x00\x00\x00"
+SIGNAL_REQUEST_BUFFER_G0 = b"\xFF\xFF\xFF\xFD"
+SIGNAL_REQUEST_BUFFER_G1 = b"\xFF\xFF\xFF\xFC"
+SIGNAL_FINISHED = b"\xFF\xFF\xFF\xFF"
+
+if PY2:
+ SIGNAL_WAS_LAST = "\x20"
+else:
+ SIGNAL_WAS_LAST = 32
+
+
+class OneWayBusyBufferingMappedFileConnection(object):
+ def __init__(self, output_path):
+ self._output_file = open(output_path, "rb+")
+ self._file_output_buffer = mmap.mmap(self._output_file.fileno(), MAPPED_FILE_SIZE, mmap.MAP_SHARED, mmap.ACCESS_WRITE)
+
+ self._out = deque()
+ self._out_size = 0
+
+ self._offset_limit = MAPPED_FILE_SIZE - 1024 * 1024 * 3
+
+ def write(self, msg):
+ self._out.append(msg)
+ self._out_size += len(msg)
+ if self._out_size > self._offset_limit:
+ self._write_buffer()
+
+ def _write_buffer(self):
+ self._file_output_buffer.seek(1, 0)
+ self._file_output_buffer.write(b"".join(self._out))
+ self._file_output_buffer.seek(0, 0)
+ self._file_output_buffer.write(b'\x01')
+
+
+class BufferingTCPMappedFileConnection(object):
+ def __init__(self, input_file, output_file, port):
+ self._input_file = open(input_file, "rb+")
+ self._output_file = open(output_file, "rb+")
+ self._file_input_buffer = mmap.mmap(self._input_file.fileno(), MAPPED_FILE_SIZE, mmap.MAP_SHARED, mmap.ACCESS_READ)
+ self._file_output_buffer = mmap.mmap(self._output_file.fileno(), MAPPED_FILE_SIZE, mmap.MAP_SHARED, mmap.ACCESS_WRITE)
+ self._socket = SOCKET.socket(family=SOCKET.AF_INET, type=SOCKET.SOCK_STREAM)
+ self._socket.connect((SOCKET.gethostbyname("localhost"), port))
+
+ self._out = deque()
+ self._out_size = 0
+
+ self._input = b""
+ self._input_offset = 0
+ self._input_size = 0
+ self._was_last = False
+
+ def write(self, msg):
+ length = len(msg)
+ if length > MAPPED_FILE_SIZE:
+ raise Exception("Serialized object does not fit into a single buffer.")
+ tmp = self._out_size + length
+ if tmp > MAPPED_FILE_SIZE:
+ self._write_buffer()
+ self.write(msg)
+ else:
+ self._out.append(msg)
+ self._out_size = tmp
+
+ def _write_buffer(self):
+ self._file_output_buffer.seek(0, 0)
+ self._file_output_buffer.write(b"".join(self._out))
+ self._socket.send(pack(">i", self._out_size))
+ self._out.clear()
+ self._out_size = 0
+ self._socket.recv(1, SOCKET.MSG_WAITALL)
+
+ def read(self, des_size, ignored=None):
+ if self._input_size == self._input_offset:
+ self._read_buffer()
+ old_offset = self._input_offset
+ self._input_offset += des_size
+ return self._input[old_offset:self._input_offset]
+
+ def _read_buffer(self):
+ self._socket.send(SIGNAL_REQUEST_BUFFER)
+ self._file_input_buffer.seek(0, 0)
+ self._input_offset = 0
+ meta_size = self._socket.recv(5, SOCKET.MSG_WAITALL)
+ self._input_size = unpack(">I", meta_size[:4])[0]
+ self._was_last = meta_size[4] == SIGNAL_WAS_LAST
+ self._input = self._file_input_buffer.read(self._input_size)
+
+ def send_end_signal(self):
+ if self._out_size:
+ self._write_buffer()
+ self._socket.send(SIGNAL_FINISHED)
+
+ def has_next(self, ignored=None):
+ return not self._was_last or not self._input_size == self._input_offset
+
+ def reset(self):
+ self._was_last = False
+ self._input_size = 0
+ self._input_offset = 0
+ self._input = b""
+
+
+class TwinBufferingTCPMappedFileConnection(BufferingTCPMappedFileConnection):
+ def __init__(self, input_file, output_file, port):
+ super(TwinBufferingTCPMappedFileConnection, self).__init__(input_file, output_file, port)
+ self._input = [b"", b""]
+ self._input_offset = [0, 0]
+ self._input_size = [0, 0]
+ self._was_last = [False, False]
+
+ def read(self, des_size, group):
+ if self._input_size[group] == self._input_offset[group]:
+ self._read_buffer(group)
+ old_offset = self._input_offset[group]
+ self._input_offset[group] += des_size
+ return self._input[group][old_offset:self._input_offset[group]]
+
+ def _read_buffer(self, group):
+ if group:
+ self._socket.send(SIGNAL_REQUEST_BUFFER_G1)
+ else:
+ self._socket.send(SIGNAL_REQUEST_BUFFER_G0)
+ self._file_input_buffer.seek(0, 0)
+ self._input_offset[group] = 0
+ meta_size = self._socket.recv(5, SOCKET.MSG_WAITALL)
+ self._input_size[group] = unpack(">I", meta_size[:4])[0]
+ self._was_last[group] = meta_size[4] == SIGNAL_WAS_LAST
+ self._input[group] = self._file_input_buffer.read(self._input_size[group])
+
+ def has_next(self, group):
+ return not self._was_last[group] or not self._input_size[group] == self._input_offset[group]
+
+ def reset(self):
+ self._was_last = [False, False]
+ self._input_size = [0, 0]
+ self._input_offset = [0, 0]
+ self._input = [b"", b""]
+
+
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Constants.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Constants.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Constants.py
new file mode 100644
index 0000000..0ca2232
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Constants.py
@@ -0,0 +1,31 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+class Types(object):
+ TYPE_TUPLE = b'\x0B'
+ TYPE_BOOLEAN = b'\x0A'
+ TYPE_BYTE = b'\x09'
+ TYPE_SHORT = b'\x08'
+ TYPE_INTEGER = b'\x07'
+ TYPE_LONG = b'\x06'
+ TYPE_DOUBLE = b'\x04'
+ TYPE_FLOAT = b'\x05'
+ TYPE_STRING = b'\x02'
+ TYPE_NULL = b'\x00'
+ TYPE_BYTES = b'\x01'
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Iterator.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Iterator.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Iterator.py
new file mode 100644
index 0000000..fb0e26d
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/Iterator.py
@@ -0,0 +1,327 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from struct import unpack
+from collections import deque
+
+try:
+ import _abcoll as defIter
+except:
+ import _collections_abc as defIter
+
+from flink.connection.Constants import Types
+
+
+class ListIterator(defIter.Iterator):
+ def __init__(self, values):
+ super(ListIterator, self).__init__()
+ self._values = deque(values)
+
+ def __next__(self):
+ return self.next()
+
+ def next(self):
+ if self.has_next():
+ return self._values.popleft()
+ else:
+ raise StopIteration
+
+ def has_next(self):
+ return self._values
+
+
+class GroupIterator(defIter.Iterator):
+ def __init__(self, iterator, keys=None):
+ super(GroupIterator, self).__init__()
+ self.iterator = iterator
+ self.key = None
+ self.keys = keys
+ if self.keys is None:
+ self._extract_keys = self._extract_keys_id
+ self.cur = None
+ self.empty = False
+
+ def _init(self):
+ if self.iterator.has_next():
+ self.empty = False
+ self.cur = self.iterator.next()
+ self.key = self._extract_keys(self.cur)
+ else:
+ self.empty = True
+
+ def __next__(self):
+ return self.next()
+
+ def next(self):
+ if self.has_next():
+ tmp = self.cur
+ if self.iterator.has_next():
+ self.cur = self.iterator.next()
+ if self.key != self._extract_keys(self.cur):
+ self.empty = True
+ else:
+ self.cur = None
+ self.empty = True
+ return tmp
+ else:
+ raise StopIteration
+
+ def has_next(self):
+ if self.empty:
+ return False
+ return self.key == self._extract_keys(self.cur)
+
+ def has_group(self):
+ return self.cur is not None
+
+ def next_group(self):
+ self.key = self._extract_keys(self.cur)
+ self.empty = False
+
+ def _extract_keys(self, x):
+ return [x[k] for k in self.keys]
+
+ def _extract_keys_id(self, x):
+ return x
+
+
+class CoGroupIterator(object):
+ NONE_REMAINED = 1
+ FIRST_REMAINED = 2
+ SECOND_REMAINED = 3
+ FIRST_EMPTY = 4
+ SECOND_EMPTY = 5
+
+ def __init__(self, c1, c2, k1, k2):
+ self.i1 = GroupIterator(c1, k1)
+ self.i2 = GroupIterator(c2, k2)
+ self.p1 = None
+ self.p2 = None
+ self.match = None
+ self.key = None
+
+ def _init(self):
+ self.i1._init()
+ self.i2._init()
+
+ def next(self):
+ first_empty = True
+ second_empty = True
+
+ if self.match != CoGroupIterator.FIRST_EMPTY:
+ if self.match == CoGroupIterator.FIRST_REMAINED:
+ first_empty = False
+ else:
+ if self.i1.has_group():
+ self.i1.next_group()
+ self.key = self.i1.key
+ first_empty = False
+
+ if self.match != CoGroupIterator.SECOND_EMPTY:
+ if self.match == CoGroupIterator.SECOND_REMAINED:
+ second_empty = False
+ else:
+ if self.i2.has_group():
+ self.i2.next_group()
+ second_empty = False
+
+ if first_empty and second_empty:
+ return False
+ elif first_empty and (not second_empty):
+ self.p1 = DummyIterator()
+ self.p2 = self.i2
+ self.match = CoGroupIterator.FIRST_EMPTY
+ return True
+ elif (not first_empty) and second_empty:
+ self.p1 = self.i1
+ self.p2 = DummyIterator()
+ self.match = CoGroupIterator.SECOND_EMPTY
+ return True
+ else:
+ if self.key == self.i2.key:
+ self.p1 = self.i1
+ self.p2 = self.i2
+ self.match = CoGroupIterator.NONE_REMAINED
+ elif self.key < self.i2.key:
+ self.p1 = self.i1
+ self.p2 = DummyIterator()
+ self.match = CoGroupIterator.SECOND_REMAINED
+ else:
+ self.p1 = DummyIterator()
+ self.p2 = self.i2
+ self.match = CoGroupIterator.FIRST_REMAINED
+ return True
+
+
+class Iterator(defIter.Iterator):
+ def __init__(self, con, group=0):
+ super(Iterator, self).__init__()
+ self._connection = con
+ self._init = True
+ self._group = group
+ self._deserializer = None
+
+ def __next__(self):
+ return self.next()
+
+ def next(self):
+ if self.has_next():
+ if self._deserializer is None:
+ self._deserializer = _get_deserializer(self._group, self._connection.read)
+ return self._deserializer.deserialize()
+ else:
+ raise StopIteration
+
+ def has_next(self):
+ return self._connection.has_next(self._group)
+
+ def _reset(self):
+ self._deserializer = None
+
+
+class DummyIterator(Iterator):
+ def __init__(self):
+ super(Iterator, self).__init__()
+
+ def __next__(self):
+ raise StopIteration
+
+ def next(self):
+ raise StopIteration
+
+ def has_next(self):
+ return False
+
+
+def _get_deserializer(group, read, type=None):
+ if type is None:
+ type = read(1, group)
+ return _get_deserializer(group, read, type)
+ elif type == Types.TYPE_TUPLE:
+ return TupleDeserializer(read, group)
+ elif type == Types.TYPE_BYTE:
+ return ByteDeserializer(read, group)
+ elif type == Types.TYPE_BYTES:
+ return ByteArrayDeserializer(read, group)
+ elif type == Types.TYPE_BOOLEAN:
+ return BooleanDeserializer(read, group)
+ elif type == Types.TYPE_FLOAT:
+ return FloatDeserializer(read, group)
+ elif type == Types.TYPE_DOUBLE:
+ return DoubleDeserializer(read, group)
+ elif type == Types.TYPE_INTEGER:
+ return IntegerDeserializer(read, group)
+ elif type == Types.TYPE_LONG:
+ return LongDeserializer(read, group)
+ elif type == Types.TYPE_STRING:
+ return StringDeserializer(read, group)
+ elif type == Types.TYPE_NULL:
+ return NullDeserializer(read, group)
+
+
+class TupleDeserializer(object):
+ def __init__(self, read, group):
+ self.read = read
+ self._group = group
+ size = unpack(">I", self.read(4, self._group))[0]
+ self.deserializer = [_get_deserializer(self._group, self.read) for _ in range(size)]
+
+ def deserialize(self):
+ return tuple([s.deserialize() for s in self.deserializer])
+
+
+class ByteDeserializer(object):
+ def __init__(self, read, group):
+ self.read = read
+ self._group = group
+
+ def deserialize(self):
+ return unpack(">c", self.read(1, self._group))[0]
+
+
+class ByteArrayDeserializer(object):
+ def __init__(self, read, group):
+ self.read = read
+ self._group = group
+
+ def deserialize(self):
+ size = unpack(">i", self.read(4, self._group))[0]
+ return bytearray(self.read(size, self._group)) if size else bytearray(b"")
+
+
+class BooleanDeserializer(object):
+ def __init__(self, read, group):
+ self.read = read
+ self._group = group
+
+ def deserialize(self):
+ return unpack(">?", self.read(1, self._group))[0]
+
+
+class FloatDeserializer(object):
+ def __init__(self, read, group):
+ self.read = read
+ self._group = group
+
+ def deserialize(self):
+ return unpack(">f", self.read(4, self._group))[0]
+
+
+class DoubleDeserializer(object):
+ def __init__(self, read, group):
+ self.read = read
+ self._group = group
+
+ def deserialize(self):
+ return unpack(">d", self.read(8, self._group))[0]
+
+
+class IntegerDeserializer(object):
+ def __init__(self, read, group):
+ self.read = read
+ self._group = group
+
+ def deserialize(self):
+ return unpack(">i", self.read(4, self._group))[0]
+
+
+class LongDeserializer(object):
+ def __init__(self, read, group):
+ self.read = read
+ self._group = group
+
+ def deserialize(self):
+ return unpack(">q", self.read(8, self._group))[0]
+
+
+class StringDeserializer(object):
+ def __init__(self, read, group):
+ self.read = read
+ self._group = group
+
+ def deserialize(self):
+ length = unpack(">i", self.read(4, self._group))[0]
+ return self.read(length, self._group).decode("utf-8") if length else ""
+
+
+class NullDeserializer(object):
+ def __init__(self, read, group):
+ self.read = read
+ self._group = group
+
+ def deserialize(self):
+ return None
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/__init__.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/__init__.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/__init__.py
new file mode 100644
index 0000000..65b48d4
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/connection/__init__.py
@@ -0,0 +1,17 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery10.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery10.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery10.py
new file mode 100644
index 0000000..032ef85
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery10.py
@@ -0,0 +1,115 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import sys
+
+from datetime import datetime
+from flink.plan.Environment import get_environment
+from flink.plan.Constants import INT, STRING, FLOAT, WriteMode
+from flink.functions.FilterFunction import FilterFunction
+from flink.functions.ReduceFunction import ReduceFunction
+from flink.functions.JoinFunction import JoinFunction
+from flink.functions.MapFunction import MapFunction
+
+class OrderFilter(FilterFunction):
+ def filter(self, value):
+ if (datetime.strptime(value[2], "%Y-%m-%d") > datetime.strptime("1990-12-31", "%Y-%m-%d")):
+ return True
+ else:
+ return False
+
+class LineitemFilter(FilterFunction):
+ def filter(self, value):
+ if value[3] == "R":
+ return True
+ else:
+ return False
+
+class ComputeRevenue(MapFunction):
+ def map(self, value):
+ return (value[0], value[1] * (1 - value[2]))
+
+class SumReducer(ReduceFunction):
+ def reduce(self, value1, value2):
+ return (value1[0], value1[1] + value2[1])
+
+class test(JoinFunction):
+ def join(self, value1, value2):
+ return (value1[1], value2[1])
+
+
+if __name__ == "__main__":
+
+ env = get_environment()
+
+ if len(sys.argv) != 6:
+ sys.exit("Usage: ./bin/pyflink.sh TPCHQuery10 <customer path> <order path> <lineitem path> <nation path> <output path>")
+
+ customer = env \
+ .read_csv(sys.argv[1],
+ [INT, STRING, STRING, INT, STRING, FLOAT, STRING, STRING], '\n', '|') \
+ .project(0,1,2,3,5)
+
+ order = env \
+ .read_csv(sys.argv[2],
+ [INT, INT, STRING, FLOAT, STRING, STRING, STRING, INT, STRING], '\n', '|') \
+ .project(0,1,4) \
+ .filter(OrderFilter()) \
+ .project(0,1)
+
+ lineitem = env \
+ .read_csv(sys.argv[3], [INT, INT, INT, INT, INT, FLOAT, FLOAT, FLOAT,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING], '\n', '|') \
+ .project(0,5,6,8) \
+ .filter(LineitemFilter()) \
+ .map(ComputeRevenue(), [INT, FLOAT])
+
+ nation = env \
+ .read_csv(sys.argv[4], [INT, STRING, INT, STRING], '\n', '|') \
+ .project(0,1)
+
+ revenueByCustomer = order \
+ .join_with_huge(lineitem) \
+ .where(0) \
+ .equal_to(0) \
+ .project_first(1) \
+ .project_second(1)
+
+ revenueByCustomer = revenueByCustomer \
+ .group_by(0) \
+ .reduce(SumReducer())
+
+ customerWithNation = customer \
+ .join_with_tiny(nation) \
+ .where(3) \
+ .equal_to(0) \
+ .project_first(0,1,2) \
+ .project_second(1) \
+ .project_first(4)
+
+ result = customerWithNation \
+ .join(revenueByCustomer) \
+ .where(0) \
+ .equal_to(0) \
+ .project_first(0,1,2,3,4) \
+ .project_second(1)
+
+ result.write_csv(sys.argv[5], '\n', '|', WriteMode.OVERWRITE)
+
+ env.set_degree_of_parallelism(1)
+
+ env.execute(local=True)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery3.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery3.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery3.py
new file mode 100644
index 0000000..5fafb01
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TPCHQuery3.py
@@ -0,0 +1,106 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import sys
+
+from datetime import datetime
+from flink.plan.Environment import get_environment
+from flink.plan.Constants import INT, STRING, FLOAT, WriteMode
+from flink.functions.FilterFunction import FilterFunction
+from flink.functions.ReduceFunction import ReduceFunction
+from flink.functions.JoinFunction import JoinFunction
+
+class CustomerFilter(FilterFunction):
+ def filter(self, value):
+ if value[1] == "AUTOMOBILE":
+ return True
+ else:
+ return False
+
+class LineitemFilter(FilterFunction):
+ def filter(self, value):
+ if (datetime.strptime(value[3], "%Y-%m-%d") > datetime.strptime("1995-03-12", "%Y-%m-%d")):
+ return True
+ else:
+ return False
+
+class OrderFilter(FilterFunction):
+ def filter(self, value):
+ if (datetime.strptime(value[2], "%Y-%m-%d") < datetime.strptime("1995-03-12", "%Y-%m-%d")):
+ return True
+ else:
+ return False
+
+class SumReducer(ReduceFunction):
+ def reduce(self, value1, value2):
+ return (value1[0], value1[1] + value2[1], value1[2], value1[3])
+
+class CustomerOrderJoin(JoinFunction):
+ def join(self, value1, value2):
+ return (value2[0], 0.0, value2[2], value2[3])
+
+class CustomerOrderLineitemJoin(JoinFunction):
+ def join(self, value1, value2):
+ return (value1[0], value2[1] * (1 - value2[2]), value1[2], value1[3])
+
+
+
+if __name__ == "__main__":
+ env = get_environment()
+
+ if len(sys.argv) != 5:
+ sys.exit("Usage: ./bin/pyflink.sh TPCHQuery3 <lineitem path> <customer path> <order path> <output path>")
+ lineitem = env \
+ .read_csv(sys.argv[1], [INT, INT, INT, INT, INT, FLOAT, FLOAT, FLOAT,
+ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING], '\n', '|') \
+ .project(0,5,6,10) \
+ .filter(LineitemFilter())
+
+
+ customer = env \
+ .read_csv(sys.argv[2],
+ [INT, STRING, STRING, INT, STRING, FLOAT, STRING, STRING], '\n', '|') \
+ .project(0,6) \
+ .filter(CustomerFilter())
+
+ order = env \
+ .read_csv(sys.argv[3],
+ [INT, INT, STRING, FLOAT, STRING, STRING, STRING, INT, STRING], '\n', '|') \
+ .project(0,1,4,7) \
+ .filter(OrderFilter())
+
+ customerWithOrder = customer \
+ .join(order) \
+ .where(0) \
+ .equal_to(1) \
+ .using(CustomerOrderJoin(),[INT, FLOAT, STRING, INT])
+
+ result = customerWithOrder \
+ .join(lineitem) \
+ .where(0) \
+ .equal_to(0) \
+ .using(CustomerOrderLineitemJoin(), [INT, FLOAT, STRING, INT]) \
+ .group_by(0, 2, 3) \
+ .reduce(SumReducer())
+
+ result.write_csv(sys.argv[4], '\n', '|', WriteMode.OVERWRITE)
+
+ env.set_degree_of_parallelism(1)
+
+ env.execute(local=True)
+
+
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TriangleEnumeration.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TriangleEnumeration.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TriangleEnumeration.py
new file mode 100644
index 0000000..2727635
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/TriangleEnumeration.py
@@ -0,0 +1,152 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.plan.Environment import get_environment
+from flink.plan.Constants import INT, Order
+from flink.functions.FlatMapFunction import FlatMapFunction
+from flink.functions.GroupReduceFunction import GroupReduceFunction
+from flink.functions.ReduceFunction import ReduceFunction
+from flink.functions.MapFunction import MapFunction
+from flink.functions.JoinFunction import JoinFunction
+
+
+class EdgeDuplicator(FlatMapFunction):
+ def flat_map(self, value, collector):
+ collector.collect((value[0], value[1]))
+ collector.collect((value[1], value[0]))
+
+
+class DegreeCounter(GroupReduceFunction):
+ def reduce(self, iterator, collector):
+ other_vertices = []
+
+ data = iterator.next()
+ edge = (data[0], data[1])
+
+ group_vertex = edge[0]
+ other_vertices.append(edge[1])
+
+ while iterator.has_next():
+ data = iterator.next()
+ edge = [data[0], data[1]]
+ other_vertex = edge[1]
+
+ contained = False
+ for v in other_vertices:
+ if v == other_vertex:
+ contained = True
+ break
+ if not contained and not other_vertex == group_vertex:
+ other_vertices.append(other_vertex)
+
+ degree = len(other_vertices)
+
+ for other_vertex in other_vertices:
+ if group_vertex < other_vertex:
+ output_edge = (group_vertex, degree, other_vertex, 0)
+ else:
+ output_edge = (other_vertex, 0, group_vertex, degree)
+ collector.collect(output_edge)
+
+
+class DegreeJoiner(ReduceFunction):
+ def reduce(self, value1, value2):
+ edge1 = [value1[0], value1[1], value1[2], value1[3]]
+ edge2 = [value2[0], value2[1], value2[2], value2[3]]
+
+ out_edge = [edge1[0], edge1[1], edge1[2], edge1[3]]
+ if edge1[1] == 0 and (not edge1[3] == 0):
+ out_edge[1] = edge2[1]
+ elif (not edge1[1] == 0) and edge1[3] == 0:
+ out_edge[3] = edge2[3]
+ return out_edge
+
+
+class EdgeByDegreeProjector(MapFunction):
+ def map(self, value):
+ if value[1] > value[3]:
+ return (value[2], value[0])
+ else:
+ return (value[0], value[2])
+
+
+class EdgeByIdProjector(MapFunction):
+ def map(self, value):
+ edge = (value[0], value[1])
+ if value[0] > value[1]:
+ return (value[1], value[0])
+ else:
+ return (value[0], value[1])
+
+
+class TriadBuilder(GroupReduceFunction):
+ def reduce(self, iterator, collector):
+ vertices = []
+
+ y = iterator.next()
+ first_edge = [y[0], y[1]]
+
+ vertices.append(first_edge[1])
+
+ while iterator.has_next():
+ x = iterator.next()
+ second_edge = [x[0], x[1]]
+ higher_vertex_id = second_edge[1]
+
+ for lowerVertexId in vertices:
+ collector.collect((first_edge[0], lowerVertexId, higher_vertex_id))
+ vertices.append(higher_vertex_id)
+
+
+class TriadFilter(JoinFunction):
+ def join(self, value1, value2):
+ return value1
+
+
+if __name__ == "__main__":
+ env = get_environment()
+ edges = env.from_elements(
+ (1, 2), (1, 3), (1, 4), (1, 5), (2, 3), (2, 5), (3, 4), (3, 7), (3, 8), (5, 6), (7, 8))
+
+ edges_with_degrees = edges \
+ .flat_map(EdgeDuplicator(), [INT, INT]) \
+ .group_by(0) \
+ .sort_group(1, Order.ASCENDING) \
+ .reduce_group(DegreeCounter(), [INT, INT, INT, INT]) \
+ .group_by(0, 2) \
+ .reduce(DegreeJoiner())
+
+ edges_by_degree = edges_with_degrees \
+ .map(EdgeByDegreeProjector(), [INT, INT])
+
+ edges_by_id = edges_by_degree \
+ .map(EdgeByIdProjector(), [INT, INT])
+
+ triangles = edges_by_degree \
+ .group_by(0) \
+ .sort_group(1, Order.ASCENDING) \
+ .reduce_group(TriadBuilder(), [INT, INT, INT]) \
+ .join(edges_by_id) \
+ .where(1, 2) \
+ .equal_to(0, 1) \
+ .using(TriadFilter(), [INT, INT, INT])
+
+ triangles.output()
+
+ env.set_degree_of_parallelism(1)
+
+ env.execute(local=True)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WebLogAnalysis.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WebLogAnalysis.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WebLogAnalysis.py
new file mode 100644
index 0000000..d571cf9
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WebLogAnalysis.py
@@ -0,0 +1,87 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import sys
+
+from datetime import datetime
+from flink.plan.Environment import get_environment
+from flink.plan.Constants import INT, STRING, FLOAT, WriteMode
+from flink.functions.CoGroupFunction import CoGroupFunction
+from flink.functions.FilterFunction import FilterFunction
+
+
+class DocumentFilter(FilterFunction):
+ def filter(self, value):
+ keywords = [" editors ", " oscillations "]
+ for kw in keywords:
+ if kw not in value[1]:
+ return False
+ return True
+
+class RankFilter(FilterFunction):
+ def filter(self, value):
+ return value[0] > 40
+
+class VisitFilter(FilterFunction):
+ def filter(self, value):
+ return datetime.strptime(value[1], "%Y-%m-%d").year == 2007
+
+class AntiJoinVisits(CoGroupFunction):
+ def co_group(self, iterator1, iterator2, collector):
+ if not iterator2.has_next():
+ while iterator1.has_next():
+ collector.collect(iterator1.next())
+
+if __name__ == "__main__":
+
+ env = get_environment()
+
+ if len(sys.argv) != 5:
+ sys.exit("Usage: ./bin/pyflink.sh WebLogAnalysis <docments path> <ranks path> <visits path> <output path>")
+
+ documents = env \
+ .read_csv(sys.argv[1], [STRING, STRING], "\n", "|") \
+ .filter(DocumentFilter()) \
+ .project(0)
+
+ ranks = env \
+ .read_csv(sys.argv[2], [INT, STRING, INT], "\n", "|") \
+ .filter(RankFilter())
+
+ visits = env \
+ .read_csv(sys.argv[3], [STRING, STRING, STRING, FLOAT, STRING, STRING, STRING, STRING, INT], "\n", "|") \
+ .project(1,2) \
+ .filter(VisitFilter()) \
+ .project(0)
+
+ docWithRanks = documents \
+ .join(ranks) \
+ .where(0) \
+ .equal_to(1) \
+ .project_second(0,1,2)
+
+ result = docWithRanks \
+ .co_group(visits) \
+ .where(1) \
+ .equal_to(0) \
+ .using(AntiJoinVisits(), [INT, STRING, INT])
+
+ result.write_csv(sys.argv[4], '\n', '|', WriteMode.OVERWRITE)
+
+ env.set_degree_of_parallelism(1)
+
+ env.execute(local=True)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WordCount.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WordCount.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WordCount.py
new file mode 100644
index 0000000..8a89a6f
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/WordCount.py
@@ -0,0 +1,61 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import sys
+
+from flink.plan.Environment import get_environment
+from flink.plan.Constants import INT, STRING
+from flink.functions.FlatMapFunction import FlatMapFunction
+from flink.functions.GroupReduceFunction import GroupReduceFunction
+
+
+class Tokenizer(FlatMapFunction):
+ def flat_map(self, value, collector):
+ for word in value.lower().split():
+ collector.collect((1, word))
+
+
+class Adder(GroupReduceFunction):
+ def reduce(self, iterator, collector):
+ count, word = iterator.next()
+ count += sum([x[0] for x in iterator])
+ collector.collect((count, word))
+
+
+if __name__ == "__main__":
+ env = get_environment()
+ if len(sys.argv) != 1 and len(sys.argv) != 3:
+ sys.exit("Usage: ./bin/pyflink.sh WordCount[ - <text path> <result path>]")
+
+ if len(sys.argv) == 3:
+ data = env.read_text(sys.argv[1])
+ else:
+ data = env.from_elements("hello","world","hello","car","tree","data","hello")
+
+ result = data \
+ .flat_map(Tokenizer(), (INT, STRING)) \
+ .group_by(1) \
+ .reduce_group(Adder(), (INT, STRING), combinable=True) \
+
+ if len(sys.argv) == 3:
+ result.write_csv(sys.argv[2])
+ else:
+ result.output()
+
+ env.set_degree_of_parallelism(1)
+
+ env.execute(local=True)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/__init__.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/__init__.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/__init__.py
new file mode 100644
index 0000000..d35bf39
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example/__init__.py
@@ -0,0 +1,17 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CoGroupFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CoGroupFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CoGroupFunction.py
new file mode 100644
index 0000000..db951fe
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CoGroupFunction.py
@@ -0,0 +1,53 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.functions import Function, RuntimeContext
+from flink.connection import Iterator, Connection, Collector
+
+
+class CoGroupFunction(Function.Function):
+ def __init__(self):
+ super(CoGroupFunction, self).__init__()
+ self._keys1 = None
+ self._keys2 = None
+
+ def _configure(self, input_file, output_file, port):
+ self._connection = Connection.TwinBufferingTCPMappedFileConnection(input_file, output_file, port)
+ self._iterator = Iterator.Iterator(self._connection, 0)
+ self._iterator2 = Iterator.Iterator(self._connection, 1)
+ self._cgiter = Iterator.CoGroupIterator(self._iterator, self._iterator2, self._keys1, self._keys2)
+ self.context = RuntimeContext.RuntimeContext(self._iterator, self._collector)
+ self._configure_chain(Collector.Collector(self._connection))
+
+ def _run(self):
+ collector = self._collector
+ iterator = self._cgiter
+ function = self.co_group
+ iterator._init()
+ while iterator.next():
+ result = function(iterator.p1, iterator.p2, collector)
+ if result is not None:
+ for res in result:
+ collector.collect(res)
+ while iterator.p1.has_next():
+ iterator.p1.next()
+ while iterator.p2.has_next():
+ iterator.p2.next()
+ collector._close()
+
+ def co_group(self, iterator1, iterator2, collector):
+ pass
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CrossFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CrossFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CrossFunction.py
new file mode 100644
index 0000000..6657d45
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/CrossFunction.py
@@ -0,0 +1,34 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.functions import Function
+
+
+class CrossFunction(Function.Function):
+ def __init__(self):
+ super(CrossFunction, self).__init__()
+
+ def _run(self):
+ collector = self._collector
+ function = self.cross
+ iterator = self._iterator
+ for value in iterator:
+ collector.collect(function(value[0], value[1]))
+ collector._close()
+
+ def cross(self, value1, value2):
+ pass
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FilterFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FilterFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FilterFunction.py
new file mode 100644
index 0000000..fb4fb74
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FilterFunction.py
@@ -0,0 +1,38 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.functions import Function
+
+
+class FilterFunction(Function.Function):
+ def __init__(self):
+ super(FilterFunction, self).__init__()
+
+ def _run(self):
+ collector = self._collector
+ function = self.filter
+ for value in self._iterator:
+ if function(value):
+ collector.collect(value)
+ collector._close()
+
+ def collect(self, value):
+ if self.filter(value):
+ self._collector.collect(value)
+
+ def filter(self, value):
+ pass
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FlatMapFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FlatMapFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FlatMapFunction.py
new file mode 100644
index 0000000..ebc493c
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/FlatMapFunction.py
@@ -0,0 +1,44 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.functions import Function
+
+
+class FlatMapFunction(Function.Function):
+ def __init__(self):
+ super(FlatMapFunction, self).__init__()
+
+ def _run(self):
+ collector = self._collector
+ function = self.flat_map
+ iterator = self._iterator
+ for value in iterator:
+ result = function(value, collector)
+ if result is not None:
+ for res in result:
+ collector.collect(res)
+ collector._close()
+
+ def collect(self, value):
+ collector = self._collector
+ result = self.flat_map(value, collector)
+ if result is not None:
+ for res in result:
+ collector.collect(res)
+
+ def flat_map(self, value, collector):
+ pass
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/Function.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/Function.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/Function.py
new file mode 100644
index 0000000..5323462
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/Function.py
@@ -0,0 +1,83 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from abc import ABCMeta, abstractmethod
+import sys
+from collections import deque
+from flink.connection import Connection, Iterator, Collector
+from flink.functions import RuntimeContext
+
+
+class Function(object):
+ __metaclass__ = ABCMeta
+
+ def __init__(self):
+ self._connection = None
+ self._iterator = None
+ self._collector = None
+ self.context = None
+ self._chain_operator = None
+
+ def _configure(self, input_file, output_file, port):
+ self._connection = Connection.BufferingTCPMappedFileConnection(input_file, output_file, port)
+ self._iterator = Iterator.Iterator(self._connection)
+ self.context = RuntimeContext.RuntimeContext(self._iterator, self._collector)
+ self._configure_chain(Collector.Collector(self._connection))
+
+ def _configure_chain(self, collector):
+ if self._chain_operator is not None:
+ self._collector = self._chain_operator
+ self._collector.context = self.context
+ self._collector._configure_chain(collector)
+ self._collector._open()
+ else:
+ self._collector = collector
+
+ def _chain(self, operator):
+ self._chain_operator = operator
+
+ @abstractmethod
+ def _run(self):
+ pass
+
+ def _open(self):
+ pass
+
+ def _close(self):
+ self._collector._close()
+
+ def _go(self):
+ self._receive_broadcast_variables()
+ self._run()
+
+ def _receive_broadcast_variables(self):
+ broadcast_count = self._iterator.next()
+ self._iterator._reset()
+ self._connection.reset()
+ for _ in range(broadcast_count):
+ name = self._iterator.next()
+ self._iterator._reset()
+ self._connection.reset()
+ bc = deque()
+ while(self._iterator.has_next()):
+ bc.append(self._iterator.next())
+ self.context._add_broadcast_variable(name, bc)
+ self._iterator._reset()
+ self._connection.reset()
+
+
+
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/GroupReduceFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/GroupReduceFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/GroupReduceFunction.py
new file mode 100644
index 0000000..11bba30
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/GroupReduceFunction.py
@@ -0,0 +1,127 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from collections import defaultdict
+from flink.functions import Function, RuntimeContext
+from flink.connection import Connection, Iterator, Collector
+from flink.plan.Constants import Order
+
+
+class GroupReduceFunction(Function.Function):
+ def __init__(self):
+ super(GroupReduceFunction, self).__init__()
+ self._keys = None
+ self._sort_ops = []
+ self._combine = False
+ self._values = []
+
+ def _configure(self, input_file, output_file, port):
+ if self._combine:
+ self._connection = Connection.BufferingTCPMappedFileConnection(input_file, output_file, port)
+ self._iterator = Iterator.Iterator(self._connection)
+ self._collector = Collector.Collector(self._connection)
+ self.context = RuntimeContext.RuntimeContext(self._iterator, self._collector)
+ self._run = self._run_combine
+ else:
+ self._connection = Connection.BufferingTCPMappedFileConnection(input_file, output_file, port)
+ self._iterator = Iterator.Iterator(self._connection)
+ self._group_iterator = Iterator.GroupIterator(self._iterator, self._keys)
+ self.context = RuntimeContext.RuntimeContext(self._iterator, self._collector)
+ self._configure_chain(Collector.Collector(self._connection))
+ self._open()
+
+ def _open(self):
+ if self._keys is None:
+ self._extract_keys = self._extract_keys_id
+
+ def _close(self):
+ self._sort_and_combine()
+ self._collector._close()
+
+ def _set_grouping_keys(self, keys):
+ self._keys = keys
+
+ def _set_sort_ops(self, ops):
+ self._sort_ops = ops
+
+ def _run(self):#reduce
+ connection = self._connection
+ collector = self._collector
+ function = self.reduce
+ iterator = self._group_iterator
+ iterator._init()
+ while iterator.has_group():
+ iterator.next_group()
+ result = function(iterator, collector)
+ if result is not None:
+ for value in result:
+ collector.collect(value)
+ collector._close()
+ connection.send_end_signal()
+
+ def _run_combine(self):#unchained combine
+ connection = self._connection
+ collector = self._collector
+ function = self.combine
+ iterator = self._iterator
+ while 1:
+ result = function(iterator, collector)
+ if result is not None:
+ for value in result:
+ collector.collect(value)
+ connection.send_end_signal()
+ connection.reset()
+
+ def collect(self, value):#chained combine
+ self._values.append(value)
+ if len(self._values) > 1000:
+ self._sort_and_combine()
+
+ def _sort_and_combine(self):
+ values = self._values
+ function = self.combine
+ collector = self._collector
+ extractor = self._extract_keys
+ grouping = defaultdict(list)
+ for value in values:
+ grouping[extractor(value)].append(value)
+ keys = list(grouping.keys())
+ keys.sort()
+ for key in keys:
+ values = grouping[key]
+ for op in reversed(self._sort_ops):
+ values.sort(key=lambda x:x[op[0]], reverse = op[1] == Order.DESCENDING)
+ result = function(Iterator.ListIterator(values), collector)
+ if result is not None:
+ for res in result:
+ collector.collect(res)
+ self._values = []
+
+ def _extract_keys(self, x):
+ return tuple([x[k] for k in self._keys])
+
+ def _extract_sort_keys(self, x):
+ return tuple(x[k] for k in self._sort_keys)
+
+ def _extract_keys_id(self, x):
+ return x
+
+ def reduce(self, iterator, collector):
+ pass
+
+ def combine(self, iterator, collector):
+ self.reduce(iterator, collector)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/824074aa/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/JoinFunction.py
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/JoinFunction.py b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/JoinFunction.py
new file mode 100644
index 0000000..0c1274b
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/functions/JoinFunction.py
@@ -0,0 +1,33 @@
+# ###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from flink.functions import Function
+
+
+class JoinFunction(Function.Function):
+ def __init__(self):
+ super(JoinFunction, self).__init__()
+
+ def _run(self):
+ collector = self._collector
+ function = self.join
+ for value in self._iterator:
+ collector.collect(function(value[0], value[1]))
+ collector._close()
+
+ def join(self, value1, value2):
+ pass