You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrql.apache.org by fe...@apache.org on 2014/08/27 21:54:52 UTC

[1/2] MRQL-45: Add support for Apache Flink

Repository: incubator-mrql
Updated Branches:
  refs/heads/master 5258de79a -> ea690387e


http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkJsonParser.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkJsonParser.java b/flink/src/main/java/org/apache/mrql/FlinkJsonParser.java
new file mode 100644
index 0000000..114be7b
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkJsonParser.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** The JSON parser */
+public class FlinkJsonParser extends JsonFormatParser implements FlinkParser {
+    String[] tags;          // split tags
+    FlinkJsonSplitter splitter;
+
+    public void open ( String file ) {
+        try {
+            splitter = new FlinkJsonSplitter(tags,file);
+        } catch (Exception e) {
+            throw new Error(e);
+        }
+    }
+
+    public void open ( FSDataInputStream fsin, long start, long end ) {
+        try {
+            splitter = new FlinkJsonSplitter(tags,fsin,start,end);
+        } catch (Exception e) {
+            throw new Error(e);
+        }
+    }
+
+    public String slice () {
+        if (splitter.hasNext()) {
+            DataOutputBuffer b = splitter.next();
+            return new String(b.getData(),0,b.getLength());
+        } else return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkJsonSplitter.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkJsonSplitter.java b/flink/src/main/java/org/apache/mrql/FlinkJsonSplitter.java
new file mode 100644
index 0000000..faaeff3
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkJsonSplitter.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.mrql.gen.*;
+import java_cup.runtime.Symbol;
+import java.util.Iterator;
+import java.io.*;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** Extract the JSON objects tagged by tags from a data split of the input stream (fsin) */
+final public class FlinkJsonSplitter implements Iterator<DataOutputBuffer> {
+    boolean in_memory;
+    FSDataInputStream fsin; // for HDFS processing
+    InputStream in;         // for in-memory processing
+    JSONLex scanner;
+    String[] tags;
+    long start;
+    long end;
+    final DataOutputBuffer buffer;
+
+    FlinkJsonSplitter ( String[] tags, FSDataInputStream fsin, long start, long end ) {
+        in_memory = false;
+        this.tags = tags;
+        this.fsin = fsin;
+        this.end = end;
+        this.buffer = new DataOutputBuffer();
+        try {
+            fsin.seek(start);
+            this.start = (start == 0) ? start : sync(start);
+            fsin.seek(this.start);
+            scanner = new JSONLex(fsin);
+        } catch ( IOException e ) {
+            System.err.println("*** Cannot parse the data split: "+fsin);
+        }
+    }
+
+    FlinkJsonSplitter ( String[] tags, String file ) {
+        in_memory = true;
+        try {
+            in = new FileInputStream(file);
+        } catch ( Exception e ) {
+            throw new Error("Cannot open the file: "+file);
+        };
+        this.tags = tags;
+        this.buffer = new DataOutputBuffer();
+        scanner = new JSONLex(in);
+    }
+
+    private long sync ( long start ) {
+        try {
+            long first_quote = -1;
+            for ( long offset = 0; ; offset++ ) {
+                char c = (char)fsin.read();
+                if (c == '\"') {
+                    if (first_quote >= 0)
+                        if ((char)fsin.read() == ':')
+                            return start+first_quote;
+                    first_quote = offset;
+                }
+            }
+        } catch (IOException ex) {
+            return (long)0;
+        }
+    }
+
+    public boolean hasNext () {
+        try {
+            if (in_memory || start+scanner.char_pos() < end)
+                if (skip())
+                    return store();
+            return false;
+        } catch (Exception e) {
+            System.err.println(e);
+            return false;
+        }
+    }
+
+    public DataOutputBuffer next () {
+        return buffer;
+    }
+
+    public void remove () { }
+
+    boolean is_start_tag ( String tagname ) {
+        if (tags == null)
+            return true;
+        for (String tag: tags)
+            if (tag.contentEquals(tagname))
+                return true;
+        return false;
+    }
+
+    /** skip until the beginning of a split element */
+    boolean skip () throws IOException {
+        while (true) {
+            Symbol s = scanner.next_token();
+            if (s.sym == jsym.EOF || (!in_memory && start+scanner.char_pos() >= end))
+                return false;
+            if (s.sym == jsym.STRING && is_start_tag((String)s.value)) {
+                String tag = (String)s.value;
+                if (scanner.next_token().sym == jsym.COLON) {
+                    buffer.reset();
+                    buffer.write('{');
+                    buffer.write('\"');
+                    for ( int i = 0; i < tag.length(); i++ )
+                        buffer.write(tag.charAt(i));
+                    buffer.write('\"');
+                    buffer.write(':');
+                    return true;
+                }
+            }
+        }
+    }
+
+    /** store one split element into the buffer; may cross split boundaries */
+    boolean store () throws IOException {
+        int nest = 0;
+        while (true) {
+            Symbol s = scanner.next_token();
+            if (s.sym == jsym.EOF)
+                return false;
+            if (s.sym == jsym.O_BEGIN || s.sym == jsym.A_BEGIN)
+                nest++;
+            else if (s.sym == jsym.O_END || s.sym == jsym.A_END)
+                nest--;
+            String text = scanner.text();
+            for ( int i = 0; i < text.length(); i++ )
+                buffer.write(text.charAt(i));
+            if (nest == 0) {
+                buffer.write('}');
+                return true;
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkLineParser.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkLineParser.java b/flink/src/main/java/org/apache/mrql/FlinkLineParser.java
new file mode 100644
index 0000000..574286a
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkLineParser.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.*;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** A parser for line-oriented, character delimited text files (such as CVS) */
+final public class FlinkLineParser extends LineParser implements FlinkParser {
+    LineReader in;
+    byte[] line;
+
+    @Override
+    public void open ( FSDataInputStream fsin, long fstart, long fend ) {
+        in_memory = false;
+        start = fstart;
+        end = fend;
+        try {
+            in = new LineReader(fsin,start,end-start+1,maxLineLength);
+            if (false && start != 0) {  // for all but the first data split, skip the first record
+                line = in.readLine();
+                if (line != null)
+                    start += line.length;
+            };
+            pos = start;
+        } catch ( IOException e ) {
+            System.err.println("*** Cannot parse the data split: "+fsin);
+            start = end;
+        }
+    }
+
+    @Override
+    public String slice () {
+        try {
+            if (in_memory)
+                return buffered_in.readLine();
+            while (pos < end) {
+                line = in.readLine();
+                if (line == null || line.length == 0)
+                    return null;
+                pos += line.length;
+                if (line.length < maxLineLength)
+                    return new String(line);
+            };
+            return null;
+        } catch ( Exception e ) {
+            System.err.println("*** Cannot slice the text: "+e);
+            return "";
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkMRQLFileInputFormat.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkMRQLFileInputFormat.java b/flink/src/main/java/org/apache/mrql/FlinkMRQLFileInputFormat.java
new file mode 100644
index 0000000..9877ceb
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkMRQLFileInputFormat.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.mrql.gen.*;
+import java.io.*;
+import org.apache.hadoop.fs.Path;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.core.fs.FileInputSplit;
+
+
+/** A superclass for all MRQL FileInputFormats */
+abstract public class FlinkMRQLFileInputFormat implements MRQLFileInputFormat {
+    public FlinkMRQLFileInputFormat () {}
+
+    private static DataSource read_directory ( String buffer ) {
+        try {
+            String[] s = buffer.split(DataSource.separator);
+            int n = Integer.parseInt(s[1]);
+            if (s[0].equals("Binary"))
+                return new BinaryDataSource(s[2],Plan.conf);
+            else if (s[0].equals("Generator"))
+                return new GeneratorDataSource(s[2],Plan.conf);
+            else if (s[0].equals("Text"))
+                return new FlinkParsedDataSource(s[3],DataSource.parserDirectory.get(s[2]),
+                                                 ((Node)Tree.parse(s[4])).children());
+            else throw new Error("Unrecognized data source: "+s[0]);
+        } catch (Exception e) {
+            throw new Error(e);
+        }
+    }
+
+    /** encode the data source directory (to be serialized and distributed to clients) */
+    public static String encode_data_sources () {
+        return DataSource.dataSourceDirectory.toString();
+    }
+
+    /** restore the data source directory */
+    public static void restore_data_sources ( String data_sources ) {
+        if (Config.local_mode)
+            return;
+        DataSource.parserDirectory = new DataSource.ParserDirectory();
+        DataSource.parserDirectory.put("xml",FlinkXMLParser.class);
+        DataSource.parserDirectory.put("json",FlinkJsonParser.class);
+        DataSource.parserDirectory.put("line",FlinkLineParser.class);
+        DataSource.dataSourceDirectory = new DataSource.DataSourceDirectory();
+        for ( String s: data_sources.split("@@@") ) {
+            String[] p = s.split("===");
+            DataSource.dataSourceDirectory.put(p[0],read_directory(p[1]));
+        }
+    }
+
+    /** the Flink input format for this input */
+    abstract public FileInputFormat<FData> inputFormat ( String path );
+
+    /** materialize the input file into a memory Bag
+     * @param path the path directory
+     * @return a Bag that contains all data
+     */
+    public Bag materialize ( final Path path ) throws IOException {
+        final FileInputFormat<FData> sf = inputFormat(path.toString());
+        final FileInputSplit[] splits = sf.createInputSplits(1);
+        if (splits.length == 0)
+            return new Bag();
+        sf.open(splits[0]);
+        return new Bag(new BagIterator () {
+                int i = 0;
+                FData d = new FData();
+                public boolean hasNext () {
+                    try {
+                        while (sf.reachedEnd() && ++i < splits.length) {
+                            sf.close();
+                            sf.open(splits[i]);
+                        };
+                        return i < splits.length;
+                    } catch (IOException ex) {
+                        throw new Error("Cannot materialize a flink data source into a bag: "+ex);
+                    }
+                }
+                public MRData next () {
+                    try {
+                        return sf.nextRecord(d).data();
+                    } catch (IOException ex) {
+                        throw new Error("Cannot materialize a flink data source into a bag: "+ex);
+                    }
+                }
+            });
+    }
+
+    /** materialize the entire dataset into a Bag
+     * @param x the DataSet in HDFS to collect values from
+     * @param strip is not used in MapReduce mode
+     * @return the Bag that contains the collected values
+     */
+    public final Bag collect ( final DataSet x, boolean strip ) throws Exception {
+        Bag res = new Bag();
+        for ( DataSource s: x.source )
+            if (s.to_be_merged)
+                res = res.union(((FlinkDataSource)s).merge());
+            else res = res.union(s.inputFormat.newInstance().materialize(new Path(s.path)));
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkOutputFormat.java b/flink/src/main/java/org/apache/mrql/FlinkOutputFormat.java
new file mode 100644
index 0000000..6a496c9
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkOutputFormat.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.IOException;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.api.common.io.BinaryOutputFormat;
+
+public class FlinkOutputFormat extends BinaryOutputFormat<FData> {
+    private static final long serialVersionUID = 1L;
+
+    @Override
+    protected void serialize ( FData data, DataOutputView out ) throws IOException {
+        data.write(out);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkParsedDataSource.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkParsedDataSource.java b/flink/src/main/java/org/apache/mrql/FlinkParsedDataSource.java
new file mode 100644
index 0000000..949f77b
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkParsedDataSource.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.mrql.gen.Trees;
+
+
+/** A data source for a text HDFS file along with the parser to parse it */
+public class FlinkParsedDataSource extends ParsedDataSource {
+    FlinkParsedDataSource ( String path,
+                            Class<? extends Parser> parser,
+                            Trees args ) {
+        this.source_num = -1;
+        this.path = path;
+        this.inputFormat = FlinkParsedInputFormat.class;
+        to_be_merged = false;
+        this.parser = parser;
+        this.args = args;
+        try {
+            dataSourceDirectory.put(path,this);
+        } catch (Exception ex) {
+            throw new Error("Cannot open file: "+path);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkParsedInputFormat.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkParsedInputFormat.java b/flink/src/main/java/org/apache/mrql/FlinkParsedInputFormat.java
new file mode 100644
index 0000000..b17127f
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkParsedInputFormat.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.*;
+import java.util.Iterator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.fs.FileInputSplit;
+
+
+/** A FileInputFormat for text files (CVS, XML, JSON, ...) */
+final public class FlinkParsedInputFormat extends FlinkMRQLFileInputFormat {
+    public FlinkParsedInputFormat () {}
+
+    public static final class ParsedInputFormat extends FileInputFormat<FData> {
+        String pathname;
+        transient FlinkParser parser;
+        transient Iterator<MRData> iter;
+        boolean eof;
+        String data_sources; // the encoded data source directory (to be serialized and distributed to clients)
+
+        public ParsedInputFormat ( String pathname ) {
+            this.pathname = pathname;
+            data_sources = encode_data_sources();
+        }
+
+        @Override
+        public void open ( FileInputSplit split ) throws IOException {
+            super.open(split);
+            restore_data_sources(data_sources);
+            ParsedDataSource ds = (ParsedDataSource)DataSource.get(split.getPath().toString(),Plan.conf);
+            try {
+                parser = (FlinkParser)ds.parser.newInstance();
+            } catch (Exception ex) {
+                throw new Error("Unrecognized parser: "+ds.parser);
+            };
+            parser.initialize(ds.args);
+            parser.open(stream,splitStart,splitStart+splitLength-1);
+            eof = false;
+        }
+
+        @Override
+        public FData nextRecord ( FData data ) throws IOException {
+            while (!eof && (iter == null || !iter.hasNext())) {
+                String line = parser.slice();
+                if (line == null) {
+                    eof = true;
+                    return null;
+                };
+                iter = parser.parse(line).iterator();
+            };
+            return new FData(iter.next());
+        }
+
+        @Override
+        public boolean reachedEnd () {
+            return eof;
+        }
+    }
+
+    /** the Flink input format for this input */
+    public FileInputFormat<FData> inputFormat ( String path ) {
+        return new ParsedInputFormat(path);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkParser.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkParser.java b/flink/src/main/java/org/apache/mrql/FlinkParser.java
new file mode 100644
index 0000000..9ac420f
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkParser.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** for a new text data source, you must implement a new parser */
+interface FlinkParser extends Parser {
+    public void open ( FSDataInputStream fsin, long start, long end );
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkXMLParser.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkXMLParser.java b/flink/src/main/java/org/apache/mrql/FlinkXMLParser.java
new file mode 100644
index 0000000..d8c61f9
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkXMLParser.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** A parser for line-oriented, character delimited text files (such as CVS) */
+final public class FlinkXMLParser extends XMLParser implements FlinkParser {
+    FlinkXMLSplitter splitter;
+
+   public void open ( String file ) {
+        try {
+            splitter = new FlinkXMLSplitter(tags,file);
+        } catch (Exception e) {
+            throw new Error(e);
+        }
+    }
+
+    public void open ( FSDataInputStream fsin, long start, long end ) {
+        try {
+            splitter = new FlinkXMLSplitter(tags,fsin,start,end);
+        } catch (Exception e) {
+            throw new Error(e);
+        }
+    }
+
+    public String slice () {
+        if (splitter.hasNext()) {
+            DataOutputBuffer b = splitter.next();
+            return new String(b.getData(),0,b.getLength());
+        } else return null;
+   }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkXMLSplitter.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkXMLSplitter.java b/flink/src/main/java/org/apache/mrql/FlinkXMLSplitter.java
new file mode 100644
index 0000000..c68211c
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkXMLSplitter.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.mrql.gen.*;
+import java.util.Iterator;
+import javax.xml.parsers.SAXParserFactory;
+import org.xml.sax.*;
+import java.io.*;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** Extract the XML elements tagged by tags from a data split of the input stream (fsin)
+ * and store them in a buffer (to be parsed by SAX).
+ */
+final public class FlinkXMLSplitter implements Iterator<DataOutputBuffer> {
+    boolean in_memory;
+    FSDataInputStream fsin; // for HDFS processing
+    BufferedReader in;      // for in-memory processing
+    String[] tags;
+    long start;
+    long end;
+    long pos;
+    StringBuffer tagname = new StringBuffer(100);
+    String start_tagname;
+    final DataOutputBuffer buffer;
+
+    FlinkXMLSplitter ( String[] tags, FSDataInputStream fsin, long start, long end ) {
+        in_memory = false;
+        this.tags = tags;
+        this.fsin = fsin;
+        this.start = start;
+        this.end = end;
+        pos = start;
+        this.buffer = new DataOutputBuffer();
+        try {
+            fsin.seek(start);
+        } catch ( IOException e ) {
+            System.err.println("*** Cannot parse the data split: "+fsin);
+        }
+    }
+
+    FlinkXMLSplitter ( String[] tags, String file ) {
+        in_memory = true;
+        try {
+            in = new BufferedReader(new InputStreamReader(new FileInputStream(file)),
+                                    100000);
+        } catch ( Exception e ) {
+            throw new Error("Cannot open the file: "+file);
+        };
+        this.tags = tags;
+        this.buffer = new DataOutputBuffer();
+    }
+
+    public boolean hasNext () {
+        try {
+            if (in_memory || pos < end)
+                if (skip())
+                    return store();
+            return false;
+        } catch (Exception e) {
+            System.err.println(e);
+            return false;
+        }
+    }
+
+    public DataOutputBuffer next () {
+        return buffer;
+    }
+
+    public void remove () { }
+
+    boolean is_start_tag () {
+        if (tags == null)
+            return true;
+        for (String tag: tags)
+            if (tag.contentEquals(tagname))
+                return true;
+        return false;
+    }
+
+    char read_tag () throws IOException {
+        tagname.setLength(0);
+        while (true) {
+            int b = in_memory ? in.read() : fsin.read();
+            if (b == -1)
+                return ' ';
+            else if (!Character.isLetterOrDigit(b) && b != ':' && b != '_')
+                return (char)b;
+            tagname.append((char)b);
+        }
+    }
+
+    /** skip until the beginning of a split element */
+    boolean skip () throws IOException {
+        while (true) {
+            int b = in_memory ? in.read() : fsin.read();
+            pos++;
+            if (b == -1 || (!in_memory && pos >= end))
+                return false;
+            else if (b == '<') {
+                b = read_tag();
+                pos += tagname.length()+1;
+                if (is_start_tag()) {
+                    buffer.reset();
+                    buffer.write('<');
+                    for ( int i = 0; i < tagname.length(); i++ )
+                        buffer.write(tagname.charAt(i));
+                    buffer.write(b);
+                    start_tagname = new String(tagname);
+                    return true;
+                }
+            }
+        }
+    }
+
+    /** store one split element into the buffer; may cross split boundaries */
+    boolean store () throws IOException {
+        while (true) {
+            int b = in_memory ? in.read() : fsin.read();
+            pos++;
+            if (b == -1)
+                return false;
+            if (b == '&') {  // don't validate external XML entities
+                buffer.write('&');buffer.write('a');buffer.write('m');buffer.write('p');buffer.write(';');
+            } else buffer.write(b);
+            if (b == '<') {
+                b = in_memory ? in.read() : fsin.read();
+                pos++;
+                buffer.write(b);
+                if (b == '/') {
+                    b = read_tag();
+                    pos += tagname.length()+1;
+                    for ( int i = 0; i < tagname.length(); i++ )
+                        buffer.write(tagname.charAt(i));
+                    buffer.write(b);
+                    if (start_tagname.contentEquals(tagname)) {
+                        while (b != '>') {
+                            b = fsin.read();
+                            pos++;
+                            buffer.write(b);
+                        };
+                        return true;
+                    }
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/LineReader.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/LineReader.java b/flink/src/main/java/org/apache/mrql/LineReader.java
new file mode 100644
index 0000000..28bb0d1
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/LineReader.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * This file is based on source code from the Hadoop Project (http://hadoop.apache.org/), licensed by the Apache
+ * Software Foundation (ASF) under the Apache License, Version 2.0. 
+ */
+
+package org.apache.mrql;
+
+import java.io.IOException;
+
+import org.apache.flink.core.fs.FSDataInputStream;
+
+public class LineReader {
+
+	private static final int CR = '\r';
+
+	private static final int LF = '\n';
+
+	private FSDataInputStream stream;
+
+	private byte[] readBuffer;
+
+	private byte[] wrapBuffer;
+
+	private long lengthLeft;
+
+	private int readPos;
+
+	private int limit;
+
+	private boolean overLimit;
+
+	public LineReader(final FSDataInputStream strm, final long start, final long length, final int buffersize)
+			throws IOException {
+		this.stream = strm;
+		this.readBuffer = new byte[buffersize];
+		this.wrapBuffer = new byte[256];
+
+		this.lengthLeft = length;
+		this.readPos = 0;
+		this.overLimit = false;
+
+		if (start != 0) {
+			strm.seek(start);
+			readLine();
+		} else {
+			fillBuffer();
+		}
+	}
+
+	private final boolean fillBuffer() throws IOException {
+
+		int toRead = this.lengthLeft > this.readBuffer.length ? this.readBuffer.length : (int) this.lengthLeft;
+		if (this.lengthLeft <= 0) {
+			toRead = this.readBuffer.length;
+			this.overLimit = true;
+		}
+
+		int read = this.stream.read(this.readBuffer, 0, toRead);
+
+		if (read == -1) {
+			this.stream.close();
+			this.stream = null;
+			return false;
+		} else {
+			this.lengthLeft -= read;
+			this.readPos = 0;
+			this.limit = read;
+			return true;
+		}
+
+	}
+
+	public void close() throws IOException {
+		this.wrapBuffer = null;
+		this.readBuffer = null;
+		if (this.stream != null) {
+			this.stream.close();
+		}
+	}
+
+	public byte[] readLine() throws IOException {
+		if (this.stream == null || this.overLimit) {
+			return null;
+		}
+
+		int curr = 0;
+		int countInWrapBuffer = 0;
+
+		while (true) {
+			if (this.readPos >= this.limit) {
+				if (!fillBuffer()) {
+					if (countInWrapBuffer > 0) {
+						byte[] tmp = new byte[countInWrapBuffer];
+						System.arraycopy(this.wrapBuffer, 0, tmp, 0, countInWrapBuffer);
+						return tmp;
+					} else {
+						return null;
+					}
+				}
+			}
+
+			int startPos = this.readPos;
+			int count = 0;
+
+			while (this.readPos < this.limit && (curr = this.readBuffer[this.readPos++]) != LF) {
+			}
+
+			// check why we dropped out
+			if (curr == LF) {
+				// line end
+				count = this.readPos - startPos - 1;
+				if (this.readPos == 1 && countInWrapBuffer > 0 && this.wrapBuffer[countInWrapBuffer - 1] == CR) {
+					countInWrapBuffer--;
+				} else if (this.readPos > startPos + 1 && this.readBuffer[this.readPos - 2] == CR) {
+					count--;
+				}
+
+				// copy to byte array
+				if (countInWrapBuffer > 0) {
+					byte[] end = new byte[countInWrapBuffer + count];
+					System.arraycopy(this.wrapBuffer, 0, end, 0, countInWrapBuffer);
+					System.arraycopy(this.readBuffer, 0, end, countInWrapBuffer, count);
+					return end;
+				} else {
+					byte[] end = new byte[count];
+					System.arraycopy(this.readBuffer, startPos, end, 0, count);
+					return end;
+				}
+			} else {
+				count = this.limit - startPos;
+
+				// buffer exhausted
+				while (this.wrapBuffer.length - countInWrapBuffer < count) {
+					// reallocate
+					byte[] tmp = new byte[this.wrapBuffer.length * 2];
+					System.arraycopy(this.wrapBuffer, 0, tmp, 0, countInWrapBuffer);
+					this.wrapBuffer = tmp;
+				}
+
+				System.arraycopy(this.readBuffer, startPos, this.wrapBuffer, countInWrapBuffer, count);
+				countInWrapBuffer += count;
+			}
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/MR_flink.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/MR_flink.java b/flink/src/main/java/org/apache/mrql/MR_flink.java
new file mode 100644
index 0000000..511cfb9
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/MR_flink.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
+import org.apache.flink.api.java.DataSet;
+
+
+/** a wrapper of a DataSet<FData> as an MRData */
+final public class MR_flink extends MRData {
+    public DataSet<FData> flink;
+
+    public MR_flink ( DataSet<FData> d ) { flink = d; }
+
+    public void materializeAll () {};
+
+    public DataSet<FData> flink () { return flink; }
+
+    final public void write ( DataOutput out ) throws IOException {
+        throw new Error("DataSets are not serializable");
+    }
+
+    public void readFields ( DataInput in ) throws IOException {
+        throw new Error("DataSets are not serializable");
+    }
+
+    public int compareTo ( MRData x ) {
+        throw new Error("DataSets cannot be compared");
+    }
+
+    public boolean equals ( Object x ) {
+        throw new Error("DataSets cannot be compared");
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/test/java/org/apache/mrql/FlinkEvaluatorLocalModeTest.java
----------------------------------------------------------------------
diff --git a/flink/src/test/java/org/apache/mrql/FlinkEvaluatorLocalModeTest.java b/flink/src/test/java/org/apache/mrql/FlinkEvaluatorLocalModeTest.java
new file mode 100644
index 0000000..9e3b030
--- /dev/null
+++ b/flink/src/test/java/org/apache/mrql/FlinkEvaluatorLocalModeTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+public class FlinkEvaluatorLocalModeTest extends EvaluatorTest {
+
+	public void tearDown() throws IOException {
+		super.tearDown();
+		Plan.clean();
+	}
+
+	@Override
+	protected Evaluator createEvaluator() throws Exception {
+		Configuration conf = null;
+
+		Config.bsp_mode = false;
+		Config.spark_mode = false;
+		Config.flink_mode = true;
+		Config.map_reduce_mode = false;
+
+		Evaluator.evaluator = new FlinkEvaluator();
+
+		Config.quiet_execution = true;
+
+		String[] args = new String[] { "-local", "-flink" };
+
+		conf = Evaluator.evaluator.new_configuration();
+		GenericOptionsParser gop = new GenericOptionsParser(conf, args);
+		conf = gop.getConfiguration();
+
+		args = gop.getRemainingArgs();
+
+		Config.hadoop_mode = true;
+		Config.testing = true;
+		Config.parse_args(args, conf);
+		
+		Evaluator.evaluator.init(conf);
+		
+		return Evaluator.evaluator;
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/test/java/org/apache/mrql/FlinkQueryLocalModeTest.java
----------------------------------------------------------------------
diff --git a/flink/src/test/java/org/apache/mrql/FlinkQueryLocalModeTest.java b/flink/src/test/java/org/apache/mrql/FlinkQueryLocalModeTest.java
new file mode 100644
index 0000000..81a122b
--- /dev/null
+++ b/flink/src/test/java/org/apache/mrql/FlinkQueryLocalModeTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+public class FlinkQueryLocalModeTest extends QueryTest {
+
+	public void tearDown() throws IOException {
+		super.tearDown();
+		Plan.clean();
+	}
+
+	@Override
+	protected Evaluator createEvaluator() throws Exception {
+		Configuration conf = new Configuration();
+
+		Config.bsp_mode = false;
+		Config.spark_mode = false;
+		Config.flink_mode = true;
+		Config.map_reduce_mode = false;
+
+		Evaluator.evaluator = new FlinkEvaluator();
+
+		Config.quiet_execution = true;
+
+		String[] args = new String[] { "-local", "-flink" };
+
+		conf = Evaluator.evaluator.new_configuration();
+		GenericOptionsParser gop = new GenericOptionsParser(conf, args);
+		conf = gop.getConfiguration();
+
+		args = gop.getRemainingArgs();
+		
+		Config.hadoop_mode = false;
+		Config.testing = true;		
+		Config.parse_args(args, conf);
+
+		Evaluator.evaluator.init(conf);
+		
+		return Evaluator.evaluator;
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 70fb038..bf5e2df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,7 +23,7 @@
   <version>0.9.4-incubating-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Apache MRQL parent POM</name>
-  <description>Apache MRQL is a query processing and optimization system for large-scale, distributed data analysis, built on top of Apache Hadoop, Hama, and Spark</description>
+  <description>Apache MRQL is a query processing and optimization system for large-scale, distributed data analysis, built on top of Apache Hadoop, Hama, Spark, and Flink</description>
   <url>http://mrql.incubator.apache.org/</url>
   <inceptionYear>2013</inceptionYear>
   <licenses>
@@ -47,8 +47,9 @@
     <hadoop.version>1.2.1</hadoop.version>
     <yarn.version>2.2.0</yarn.version>
     <hama.version>0.6.3</hama.version>
-    <spark.version>1.0.0</spark.version>
+    <spark.version>1.0.2</spark.version>
     <scala.version>2.10</scala.version>
+    <flink.version>0.6-incubating</flink.version>
     <skipTests>true</skipTests>
   </properties>
 
@@ -58,6 +59,7 @@
     <module>mapreduce</module>
     <module>bsp</module>
     <module>spark</module>
+    <module>flink</module>
     <module>dist</module>
   </modules>
 

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen b/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
index 23c29f0..b7f43b4 100644
--- a/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
+++ b/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
@@ -97,7 +97,7 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
 
     final public void initialize_query () {
         Plan.distribute_compiled_arguments(Plan.conf);
-        if (Config.compile_functional_arguments)
+        if (spark_context != null && Config.compile_functional_arguments)
             spark_context.addJar(Plan.conf.get("mrql.jar.path"));
     }
 
@@ -169,21 +169,21 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
     @Override
     final public void dump ( String file, Tree type, MRData data ) throws Exception {
         if (data instanceof MR_dataset && ((MR_dataset)data).dataset().source.get(0) instanceof RDDDataSource) {
-	    Path path = new Path(file);
-	    FileSystem fs = path.getFileSystem(Plan.conf);
-	    PrintStream ftp = new PrintStream(fs.create(path.suffix(".type")));
+            Path path = new Path(file);
+            FileSystem fs = path.getFileSystem(Plan.conf);
+            PrintStream ftp = new PrintStream(fs.create(path.suffix(".type")));
             fs.delete(path,true);
-	    ftp.print("2@"+type.toString()+"\n");
-	    ftp.close();
+            ftp.print("2@"+type.toString()+"\n");
+            ftp.close();
             // in Spark mode, a dataset has always one RDD datasource
-	    JavaRDD<MRData> rdd = ((RDDDataSource)((MR_dataset)data).dataset().source.get(0)).rdd.cache();
+            JavaRDD<MRData> rdd = ((RDDDataSource)((MR_dataset)data).dataset().source.get(0)).rdd.cache();
             rdd.mapToPair(new PairFunction<MRData,MRContainer,MRContainer>() {
                     final MRContainer zero = new MRContainer(new MR_int(0));
                     public Tuple2<MRContainer,MRContainer> call ( MRData value ) {
                         return new Tuple2<MRContainer,MRContainer>(zero,new MRContainer(value));
                     }
                 }).saveAsHadoopFile(file,MRContainer.class,MRContainer.class,SequenceFileOutputFormat.class);
-	} else super.dump(file,type,data);
+        } else super.dump(file,type,data);
     }
 
     /** dump MRQL data into text a CVS file */
@@ -191,26 +191,26 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
     final public void dump_text ( String file, Tree type, MRData data ) throws Exception {
         if (data instanceof MR_dataset && ((MR_dataset)data).dataset().source.get(0) instanceof RDDDataSource) {
             // in Spark mode, a dataset has always one RDD datasource
-	    JavaRDD<MRData> rdd = ((RDDDataSource)((MR_dataset)data).dataset().source.get(0)).rdd.cache();
-	    int ps = Config.max_bag_size_print;
-	    Config.max_bag_size_print = -1;
-	    match type {
-	    case `T(tuple(...tps)):
-		final Trees ts = tps;
-		rdd.map(new Function<MRData,String>() {
+            JavaRDD<MRData> rdd = ((RDDDataSource)((MR_dataset)data).dataset().source.get(0)).rdd.cache();
+            int ps = Config.max_bag_size_print;
+            Config.max_bag_size_print = -1;
+            match type {
+            case `T(tuple(...tps)):
+                final Trees ts = tps;
+                rdd.map(new Function<MRData,String>() {
                         public String call ( MRData value ) {
-			    Tuple t = (Tuple)value;
-			    String s = Printer.print(t.get((short)0),ts.nth(0));
-			    for ( short i = 1; i < t.size(); i++ )
-				s += ","+Printer.print(t.get(i),ts.nth(i));
-			    return s;
-			}
-		    }).saveAsTextFile(file);
-	    case _:
-		rdd.saveAsTextFile(file);
-	    };
-	    Config.max_bag_size_print = ps;
-	} else super.dump_text(file,type,data);
+                            Tuple t = (Tuple)value;
+                            String s = Printer.print(t.get((short)0),ts.nth(0));
+                            for ( short i = 1; i < t.size(); i++ )
+                                s += ","+Printer.print(t.get(i),ts.nth(i));
+                            return s;
+                        }
+                    }).saveAsTextFile(file);
+            case _:
+                rdd.saveAsTextFile(file);
+            };
+            Config.max_bag_size_print = ps;
+        } else super.dump_text(file,type,data);
     }
 
     private static Function2<MRData,MRData,MRData> accumulator ( final Tree acc_fnc, final Environment env ) {
@@ -554,7 +554,7 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
                             }
                         });
             case GroupByJoin(`kx,`ky,`gx,`gy,`acc,`zero,`r,`x,`y,`o):
-                final int n = (int)Math.floor(Math.sqrt(Config.nodes));
+                final int n = Math.max(2,(int)Math.floor(Math.sqrt(Config.nodes)));
                 final int m = n;
                 // System.err.println("Using a groupBy join on a "+n+"*"+m+" grid of partitions");
                 final org.apache.mrql.Function fkx = evalF(kx,env);

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/spark/src/test/java/org/apache/mrql/SparkEvaluatorInMemoryTest.java
----------------------------------------------------------------------
diff --git a/spark/src/test/java/org/apache/mrql/SparkEvaluatorInMemoryTest.java b/spark/src/test/java/org/apache/mrql/SparkEvaluatorInMemoryTest.java
deleted file mode 100644
index 39c4ad4..0000000
--- a/spark/src/test/java/org/apache/mrql/SparkEvaluatorInMemoryTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.mrql;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class SparkEvaluatorInMemoryTest  extends EvaluatorTest {
-
-	@Override
-	protected Evaluator createEvaluator() throws Exception {
-		Configuration conf = null;
-
-		Config.bsp_mode = false;
-		Config.spark_mode = true;
-		Config.map_reduce_mode = false;
-
-		Evaluator.evaluator = new SparkEvaluator();
-
-		Config.quiet_execution = true;		
-		Config.hadoop_mode = false;
-		Config.testing = true;
-		Config.parse_args(new String[] {"-spark"}, conf);
-		
-		Evaluator.evaluator.init(conf);
-		
-		return Evaluator.evaluator;
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/spark/src/test/java/org/apache/mrql/SparkQueryInMemoryTest.java
----------------------------------------------------------------------
diff --git a/spark/src/test/java/org/apache/mrql/SparkQueryInMemoryTest.java b/spark/src/test/java/org/apache/mrql/SparkQueryInMemoryTest.java
deleted file mode 100644
index b35365d..0000000
--- a/spark/src/test/java/org/apache/mrql/SparkQueryInMemoryTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.mrql;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class SparkQueryInMemoryTest extends QueryTest {
-
-	@Override
-	protected Evaluator createEvaluator() throws Exception {
-		Configuration conf = null;
-
-		Config.bsp_mode = false;
-		Config.spark_mode = true;
-		Config.map_reduce_mode = false;
-
-		Evaluator.evaluator = new SparkEvaluator();
-
-		Config.quiet_execution = true;
-		Config.hadoop_mode = false;
-		Config.testing = true;
-		Config.parse_args(new String[] { "-spark" }, conf);
-		
-		Evaluator.evaluator.init(conf);
-		
-		return Evaluator.evaluator;
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/spark/src/test/java/org/apache/mrql/SparkQueryLocalModeTest.java
----------------------------------------------------------------------
diff --git a/spark/src/test/java/org/apache/mrql/SparkQueryLocalModeTest.java b/spark/src/test/java/org/apache/mrql/SparkQueryLocalModeTest.java
index 72b32e4..7c2812d 100644
--- a/spark/src/test/java/org/apache/mrql/SparkQueryLocalModeTest.java
+++ b/spark/src/test/java/org/apache/mrql/SparkQueryLocalModeTest.java
@@ -54,7 +54,7 @@ public class SparkQueryLocalModeTest extends QueryTest {
 		Config.parse_args(args, conf);
 		
 		Evaluator.evaluator.init(conf);
-		
+
 		return Evaluator.evaluator;
 	}
 }


[2/2] git commit: MRQL-45: Add support for Apache Flink

Posted by fe...@apache.org.
MRQL-45: Add support for Apache Flink


Project: http://git-wip-us.apache.org/repos/asf/incubator-mrql/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mrql/commit/ea690387
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mrql/tree/ea690387
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mrql/diff/ea690387

Branch: refs/heads/master
Commit: ea690387ea1db9f7924ffe0670bb9e167b16c83c
Parents: 5258de7
Author: fegaras <fe...@cse.uta.edu>
Authored: Wed Aug 27 14:53:49 2014 -0500
Committer: fegaras <fe...@cse.uta.edu>
Committed: Wed Aug 27 14:53:49 2014 -0500

----------------------------------------------------------------------
 bin/mrql.flink                                  |  58 ++
 conf/mrql-env.sh                                |  28 +-
 .../org/apache/mrql/AlgebraicOptimization.gen   |   8 +-
 core/src/main/java/org/apache/mrql/Config.java  |   7 +
 core/src/main/java/org/apache/mrql/Inv.java     |   2 +
 .../main/java/org/apache/mrql/LineParser.gen    |   2 +-
 .../main/java/org/apache/mrql/MRContainer.java  |   2 +-
 core/src/main/java/org/apache/mrql/MR_bool.java |   2 +
 core/src/main/java/org/apache/mrql/MR_byte.java |   3 +
 core/src/main/java/org/apache/mrql/MR_char.java |   2 +
 .../main/java/org/apache/mrql/MR_double.java    |   2 +
 .../src/main/java/org/apache/mrql/MR_float.java |   2 +
 core/src/main/java/org/apache/mrql/MR_int.java  |   2 +
 core/src/main/java/org/apache/mrql/MR_long.java |   2 +
 .../src/main/java/org/apache/mrql/MR_short.java |   2 +
 .../main/java/org/apache/mrql/MR_string.java    |   2 +
 core/src/main/java/org/apache/mrql/Main.java    |   9 +-
 .../java/org/apache/mrql/ParsedDataSource.java  |   2 +
 .../java/org/apache/mrql/PlanGeneration.gen     |   8 +
 .../java/org/apache/mrql/SystemFunctions.java   |  21 +-
 core/src/main/java/org/apache/mrql/Tuple.java   |   2 +
 core/src/main/java/org/apache/mrql/Union.java   |   2 +
 .../main/java/org/apache/mrql/XMLParser.java    |   2 +-
 .../test/java/org/apache/mrql/QueryTest.java    |   2 +-
 flink/pom.xml                                   | 148 ++++
 flink/src/main/java/org/apache/mrql/FData.java  |  73 ++
 .../java/org/apache/mrql/FDataContainer.java    |  39 ++
 .../org/apache/mrql/FlinkBinaryInputFormat.java |  44 ++
 .../java/org/apache/mrql/FlinkDataSource.java   | 290 ++++++++
 .../java/org/apache/mrql/FlinkEvaluator.gen     | 691 +++++++++++++++++++
 .../apache/mrql/FlinkGeneratorInputFormat.java  |  28 +
 .../java/org/apache/mrql/FlinkJsonParser.java   |  51 ++
 .../java/org/apache/mrql/FlinkJsonSplitter.java | 153 ++++
 .../java/org/apache/mrql/FlinkLineParser.java   |  67 ++
 .../apache/mrql/FlinkMRQLFileInputFormat.java   | 118 ++++
 .../java/org/apache/mrql/FlinkOutputFormat.java |  31 +
 .../org/apache/mrql/FlinkParsedDataSource.java  |  40 ++
 .../org/apache/mrql/FlinkParsedInputFormat.java |  85 +++
 .../main/java/org/apache/mrql/FlinkParser.java  |  26 +
 .../java/org/apache/mrql/FlinkXMLParser.java    |  50 ++
 .../java/org/apache/mrql/FlinkXMLSplitter.java  | 165 +++++
 .../main/java/org/apache/mrql/LineReader.java   | 164 +++++
 .../src/main/java/org/apache/mrql/MR_flink.java |  51 ++
 .../mrql/FlinkEvaluatorLocalModeTest.java       |  61 ++
 .../apache/mrql/FlinkQueryLocalModeTest.java    |  61 ++
 pom.xml                                         |   6 +-
 .../java/org/apache/mrql/SparkEvaluator.gen     |  56 +-
 .../apache/mrql/SparkEvaluatorInMemoryTest.java |  43 --
 .../org/apache/mrql/SparkQueryInMemoryTest.java |  43 --
 .../apache/mrql/SparkQueryLocalModeTest.java    |   2 +-
 50 files changed, 2626 insertions(+), 134 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/bin/mrql.flink
----------------------------------------------------------------------
diff --git a/bin/mrql.flink b/bin/mrql.flink
new file mode 100755
index 0000000..9a1c272
--- /dev/null
+++ b/bin/mrql.flink
@@ -0,0 +1,58 @@
+#!/bin/bash
+#--------------------------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#--------------------------------------------------------------------------------
+#
+# run Apache MRQL in Apache Flink mode
+#
+#--------------------------------------------------------------------------------
+
+MRQL_HOME="$(cd `dirname $0`/..; pwd -P)"
+
+. "$MRQL_HOME/conf/mrql-env.sh"
+
+GEN_JAR=`ls "$MRQL_HOME"/lib/mrql-gen-*.jar`
+CORE_JAR=`ls "$MRQL_HOME"/lib/mrql-core-*.jar`
+MRQL_JAR=`ls "$MRQL_HOME"/lib/mrql-flink-*.jar`
+FULL_JAR="$MRQL_HOME/lib/mrql-all-flink.jar"
+
+export FLINK_HOME FLINK_JAR FLINK_MASTER
+
+if (! [ -a $FULL_JAR ]); then
+   rm -rf "$MRQL_HOME/tmp/classes"
+   mkdir -p "$MRQL_HOME/tmp/classes"
+   pushd $MRQL_HOME/tmp/classes > /dev/null
+   $JAVA_HOME/bin/jar xf $CUP_JAR
+   $JAVA_HOME/bin/jar xf $JLINE_JAR
+   $JAVA_HOME/bin/jar xf $GEN_JAR
+   $JAVA_HOME/bin/jar xf $CORE_JAR
+   $JAVA_HOME/bin/jar xf $MRQL_JAR
+   cd ..
+   $JAVA_HOME/bin/jar cf $FULL_JAR -C classes/ .
+   popd > /dev/null
+fi
+
+if [ "$1" == "-dist" ]; then
+    args=" -flink $*"
+    args=${args// /\!}
+    $FLINK_HOME/bin/flink run $FULL_JAR -c org.apache.mrql.Main -v args $args
+else 
+    $JAVA_HOME/bin/java -classpath "$FULL_JAR:$FLINK_JAR" org.apache.mrql.Main -flink $*
+fi

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/conf/mrql-env.sh
----------------------------------------------------------------------
diff --git a/conf/mrql-env.sh b/conf/mrql-env.sh
index 90fa261..1635334 100644
--- a/conf/mrql-env.sh
+++ b/conf/mrql-env.sh
@@ -46,7 +46,7 @@ JLINE_JAR=${HOME}/.m2/repository/jline/jline/1.0/jline-1.0.jar
 
 
 # Required: Hadoop configuration. Supports versions 1.x and 2.x (YARN)
-HADOOP_VERSION=1.2.1
+HADOOP_VERSION=2.2.0
 # The Hadoop installation directory
 HADOOP_HOME=${HOME}/hadoop-${HADOOP_VERSION}
 # The Hadoop job tracker (as defined in mapred-site.xml)
@@ -64,15 +64,17 @@ BSP_MASTER_ADDRESS=localhost:40000
 HAMA_ZOOKEEPER_QUORUM=localhost
 
 
-# Optional: Spark configuration. Supports version 1.0.0 only
+# Optional: Spark configuration. Supports versions 1.0.0 and 1.0.2 only
 # (Spark versions 0.8.1, 0.9.0, and 0.9.1 are supported by MRQL 0.9.0)
 # Use the Spark prebuilts bin-hadoop1 or bin-hadoop2 (Yarn)
 # Tested in local, standalone deploy, and Yarn modes
-SPARK_HOME=${HOME}/spark-1.0.0-bin-hadoop1
-# URI of the Spark master node (to run Spark on a YARN cluster, set it to "yarn-client")
-SPARK_MASTER=spark://crete:7077
+SPARK_HOME=${HOME}/spark-1.0.2-bin-hadoop2
+# URI of the Spark master node:
+#   to run Spark on Standalone Mode, set it to spark://`hostname`:7077
+#   to run Spark on a YARN cluster, set it to "yarn-client"
+SPARK_MASTER=yarn-client
 # For a Yarn cluster set it to the number of workers to start on,
-#  for local/standalone set it to 1
+#   for local/standalone set it to 1
 SPARK_WORKER_INSTANCES=1
 # Number of cores for the workers
 SPARK_WORKER_CORES=2
@@ -81,10 +83,24 @@ SPARK_WORKER_MEMORY=1G
 # Memory for Master (e.g. 1000M, 2G)
 SPARK_MASTER_MEMORY=512M
 
+
+# Optional: Flink configuration. Supports version 0.6-incubating
+# Note: for yarn, set yarn.nodemanager.vmem-check-enabled to false in yarn-site.xml
+FLINK_VERSION=yarn-0.6-incubating
+# Flink installation directory
+FLINK_HOME=${HOME}/flink-${FLINK_VERSION}
+# Flink JobManager (on a yarn cluster, it is printed when you run yarn-session.sh)
+if [ "$FLINK_MASTER" = "" ]; then
+    FLINK_MASTER=`hostname`
+fi
+
+
 # Claspaths
 
 HAMA_JAR=${HAMA_HOME}/hama-core-${HAMA_VERSION}.jar
 
+FLINK_JAR=`ls ${FLINK_HOME}/lib/flink-*.jar`
+
 # YARN-enabled assembly jar
 if [[ -d ${SPARK_HOME}/assembly/target ]]; then
    # old Spark (0.x)

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/AlgebraicOptimization.gen
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/AlgebraicOptimization.gen b/core/src/main/java/org/apache/mrql/AlgebraicOptimization.gen
index 9126de6..b89dd3c 100644
--- a/core/src/main/java/org/apache/mrql/AlgebraicOptimization.gen
+++ b/core/src/main/java/org/apache/mrql/AlgebraicOptimization.gen
@@ -69,13 +69,17 @@ public class AlgebraicOptimization extends Simplification {
         case mapReduce2(`mx,`my,`r,cmap(lambda(`v,`b),`X),`Y,`o):
             match X {
             case groupBy(...): ;
-            case _: return translate(#<mapReduce2(lambda(`v,cmap(`mx,`b)),`my,`r,`X,`Y,`o)>);
+            case _:
+                Tree nmx = simplify(#<cmap(`mx,`b)>);
+                return translate(#<mapReduce2(lambda(`v,`nmx),`my,`r,`X,`Y,`o)>);
             };
             fail
         case mapReduce2(`mx,`my,`r,`X,cmap(lambda(`v,`b),`Y),`o):
             match Y {
             case groupBy(...): ;
-            case _: return translate(#<mapReduce2(`mx,lambda(`v,cmap(`my,`b)),`r,`X,`Y,`o)>);
+            case _:
+                Tree nmy = simplify(#<cmap(`my,`b)>);
+                return translate(#<mapReduce2(`mx,lambda(`v,`nmy),`r,`X,`Y,`o)>);
             };
             fail
         case crossProduct(`mx,`my,`r,cmap(lambda(`vx,`bx),`X),cmap(lambda(`vy,`by),`Y)):

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/Config.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/Config.java b/core/src/main/java/org/apache/mrql/Config.java
index f46319a..4c4aff9 100644
--- a/core/src/main/java/org/apache/mrql/Config.java
+++ b/core/src/main/java/org/apache/mrql/Config.java
@@ -39,6 +39,8 @@ final public class Config {
     public static boolean bsp_mode = false;
     // true, for Spark mode
     public static boolean spark_mode = false;
+    // true, for Flink mode
+    public static boolean flink_mode = false;
     // if true, it process the input interactively
     public static boolean interactive = true;
     // compile the MR functional arguments to Java bytecode at run-time
@@ -91,6 +93,7 @@ final public class Config {
         conf.setBoolean("mrql.map.reduce.mode",map_reduce_mode);
         conf.setBoolean("mrql.bsp.mode",bsp_mode);
         conf.setBoolean("mrql.spark.mode",spark_mode);
+        conf.setBoolean("mrql.flink.mode",flink_mode);
         conf.setBoolean("mrql.interactive",interactive);
         conf.setBoolean("mrql.compile.functional.arguments",compile_functional_arguments);
         conf.setBoolean("mrql.trace",trace);
@@ -121,6 +124,7 @@ final public class Config {
         map_reduce_mode = conf.getBoolean("mrql.map.reduce.mode",map_reduce_mode);
         bsp_mode = conf.getBoolean("mrql.bsp.mode",bsp_mode);
         spark_mode = conf.getBoolean("mrql.spark.mode",spark_mode);
+        flink_mode = conf.getBoolean("mrql.flink.mode",flink_mode);
         interactive = conf.getBoolean("mrql.interactive",interactive);
         compile_functional_arguments = conf.getBoolean("mrql.compile.functional.arguments",compile_functional_arguments);
         trace = conf.getBoolean("mrql.trace",trace);
@@ -170,6 +174,9 @@ final public class Config {
             } else if (args[i].equals("-spark")) {
                 spark_mode = true;
                 i++;
+            } else if (args[i].equals("-flink")) {
+                flink_mode = true;
+                i++;
             } else if (args[i].equals("-bsp_tasks")) {
                 if (++i >= args.length && Integer.parseInt(args[i]) < 1)
                     throw new Error("Expected max number of bsp tasks > 1");

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/Inv.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/Inv.java b/core/src/main/java/org/apache/mrql/Inv.java
index 3faa7fe..99db099 100644
--- a/core/src/main/java/org/apache/mrql/Inv.java
+++ b/core/src/main/java/org/apache/mrql/Inv.java
@@ -25,6 +25,8 @@ import java.io.DataOutput;
 final public class Inv extends MRData {
     private MRData value;
 
+    public Inv () {}
+
     Inv ( MRData v ) { value = v; }
 
     public void materializeAll () { value.materializeAll(); };

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/LineParser.gen
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/LineParser.gen b/core/src/main/java/org/apache/mrql/LineParser.gen
index fb1e068..5619b3f 100644
--- a/core/src/main/java/org/apache/mrql/LineParser.gen
+++ b/core/src/main/java/org/apache/mrql/LineParser.gen
@@ -26,7 +26,7 @@ import org.apache.hadoop.util.LineReader;
 
 
 /** A parser for line-oriented, character delimited text files (such as CVS) */
-final public class LineParser implements Parser {
+public class LineParser implements Parser {
     final static int maxLineLength = 1000;
     boolean in_memory;
     FSDataInputStream fsin;     // for HDFS processing

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MRContainer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MRContainer.java b/core/src/main/java/org/apache/mrql/MRContainer.java
index d72c452..42caa5c 100644
--- a/core/src/main/java/org/apache/mrql/MRContainer.java
+++ b/core/src/main/java/org/apache/mrql/MRContainer.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.io.WritableComparable;
 
 
 /** A container for MRData that implements read (the deserializer) */
-final public class MRContainer implements WritableComparable<MRContainer>, Serializable {
+public class MRContainer implements WritableComparable<MRContainer>, Serializable {
     transient MRData data;
 
     public final static byte BOOLEAN = 0, BYTE = 1, SHORT = 2, INT = 3, LONG = 4,

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_bool.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_bool.java b/core/src/main/java/org/apache/mrql/MR_bool.java
index d40573f..ef3e453 100644
--- a/core/src/main/java/org/apache/mrql/MR_bool.java
+++ b/core/src/main/java/org/apache/mrql/MR_bool.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.fs.*;
 final public class MR_bool extends MRData {
     private boolean value;
 
+    public MR_bool () {}
+
     public MR_bool ( boolean i ) { value = i; }
 
     public void materializeAll () {};

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_byte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_byte.java b/core/src/main/java/org/apache/mrql/MR_byte.java
index e5c319e..489843d 100644
--- a/core/src/main/java/org/apache/mrql/MR_byte.java
+++ b/core/src/main/java/org/apache/mrql/MR_byte.java
@@ -28,7 +28,10 @@ import org.apache.hadoop.fs.*;
 final public class MR_byte extends MRData {
     private byte value;
 
+    public MR_byte () {}
+
     public MR_byte ( byte i ) { value = i; }
+
     public MR_byte ( int i ) { value = (byte)i; }
 
     public void materializeAll () {};

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_char.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_char.java b/core/src/main/java/org/apache/mrql/MR_char.java
index f0e9721..7a70996 100644
--- a/core/src/main/java/org/apache/mrql/MR_char.java
+++ b/core/src/main/java/org/apache/mrql/MR_char.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
 final public class MR_char extends MRData {
     private char value;
 
+    public MR_char () {}
+
     public MR_char ( char i ) { value = i; }
 
     public void materializeAll () {};

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_double.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_double.java b/core/src/main/java/org/apache/mrql/MR_double.java
index 567917c..c0e7d57 100644
--- a/core/src/main/java/org/apache/mrql/MR_double.java
+++ b/core/src/main/java/org/apache/mrql/MR_double.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
 final public class MR_double extends MRData {
     private double value;
 
+    public MR_double () {}
+
     public MR_double ( double x ) { value = x; }
 
     public void materializeAll () {};

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_float.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_float.java b/core/src/main/java/org/apache/mrql/MR_float.java
index d8dda38..121a37c 100644
--- a/core/src/main/java/org/apache/mrql/MR_float.java
+++ b/core/src/main/java/org/apache/mrql/MR_float.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
 final public class MR_float extends MRData {
     private float value;
 
+    public MR_float () {}
+
     public MR_float ( float x ) { value = x; }
 
     public MR_float ( double x ) { value = (float)x; }

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_int.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_int.java b/core/src/main/java/org/apache/mrql/MR_int.java
index 6586fe0..37e5d9e 100644
--- a/core/src/main/java/org/apache/mrql/MR_int.java
+++ b/core/src/main/java/org/apache/mrql/MR_int.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.fs.*;
 final public class MR_int extends MRData {
     private int value;
 
+    public MR_int () {}
+
     public MR_int ( int i ) { value = i; }
 
     public void materializeAll () {};

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_long.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_long.java b/core/src/main/java/org/apache/mrql/MR_long.java
index 2078048..e16f2e7 100644
--- a/core/src/main/java/org/apache/mrql/MR_long.java
+++ b/core/src/main/java/org/apache/mrql/MR_long.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
 final public class MR_long extends MRData {
     private long value;
 
+    public MR_long () {}
+
     public MR_long ( long i ) { value = i; }
 
     public void materializeAll () {};

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_short.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_short.java b/core/src/main/java/org/apache/mrql/MR_short.java
index c50e124..b6e34a8 100644
--- a/core/src/main/java/org/apache/mrql/MR_short.java
+++ b/core/src/main/java/org/apache/mrql/MR_short.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
 final public class MR_short extends MRData {
     private short value;
 
+    public MR_short () {}
+
     public MR_short ( short i ) { value = i; }
 
     public void materializeAll () {};

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_string.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_string.java b/core/src/main/java/org/apache/mrql/MR_string.java
index a914cca..54cdd59 100644
--- a/core/src/main/java/org/apache/mrql/MR_string.java
+++ b/core/src/main/java/org/apache/mrql/MR_string.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
 final public class MR_string extends MRData {
     private String value;
 
+    public MR_string () {}
+
     public MR_string ( String s ) { value = s; }
 
     public void materializeAll () {};

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/Main.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/Main.java b/core/src/main/java/org/apache/mrql/Main.java
index b5da0f4..5607a19 100644
--- a/core/src/main/java/org/apache/mrql/Main.java
+++ b/core/src/main/java/org/apache/mrql/Main.java
@@ -61,6 +61,8 @@ final public class Main {
             Evaluator.evaluator = (Evaluator)Class.forName("org.apache.mrql.BSPEvaluator").newInstance();
         else if (Config.spark_mode)
             Evaluator.evaluator = (Evaluator)Class.forName("org.apache.mrql.SparkEvaluator").newInstance();
+        else if (Config.flink_mode)
+            Evaluator.evaluator = (Evaluator)Class.forName("org.apache.mrql.FlinkEvaluator").newInstance();
         else // when Config.map_reduce_mode but also the default
             Evaluator.evaluator = (Evaluator)Class.forName("org.apache.mrql.MapReduceEvaluator").newInstance();
     }
@@ -77,12 +79,15 @@ final public class Main {
 
     public static void main ( String[] args ) throws Exception {
         Config.hadoop_mode = false;
+        if (args.length == 2 && args[0].equals("args"))  // needed for mrql.flink script
+            args = args[1].substring(1).split("!");
         for ( String arg: args ) {
             Config.hadoop_mode |= arg.equals("-local") || arg.equals("-dist");
             Config.bsp_mode |= arg.equals("-bsp");
             Config.spark_mode |= arg.equals("-spark");
+            Config.flink_mode |= arg.equals("-flink");
         };
-        Config.map_reduce_mode = !Config.bsp_mode && !Config.spark_mode;
+        Config.map_reduce_mode = !Config.bsp_mode && !Config.spark_mode && !Config.flink_mode;
         initialize_evaluator();
 	if (Config.hadoop_mode) {
 	    conf = Evaluator.evaluator.new_configuration();
@@ -110,6 +115,8 @@ final public class Main {
                 System.out.print("distributed ");
             if (Config.spark_mode)
                 System.out.println("Spark mode using "+Config.nodes+" tasks)");
+            else if (Config.flink_mode)
+                System.out.println("Flink mode using "+Config.nodes+" tasks)");
             else if (Config.bsp_mode)
                 System.out.println("Hama BSP mode over "+Config.nodes+" BSP tasks)");
             else if (Config.nodes > 0)

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/ParsedDataSource.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/ParsedDataSource.java b/core/src/main/java/org/apache/mrql/ParsedDataSource.java
index e000405..5b5a06d 100644
--- a/core/src/main/java/org/apache/mrql/ParsedDataSource.java
+++ b/core/src/main/java/org/apache/mrql/ParsedDataSource.java
@@ -28,6 +28,8 @@ public class ParsedDataSource extends DataSource {
     public Class<? extends Parser> parser;
     public Trees args;
 
+    ParsedDataSource () {}
+
     ParsedDataSource ( int source_num,
                        String path,
                        Class<? extends Parser> parser,

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/PlanGeneration.gen
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/PlanGeneration.gen b/core/src/main/java/org/apache/mrql/PlanGeneration.gen
index fe1a1d9..72200fc 100644
--- a/core/src/main/java/org/apache/mrql/PlanGeneration.gen
+++ b/core/src/main/java/org/apache/mrql/PlanGeneration.gen
@@ -672,6 +672,12 @@ final public class PlanGeneration extends AlgebraicOptimization {
         match plan {
         case MapReduce2(`mx,`my,`r,`x,`y,`o):
             return physical_plan(#<MapAggregateReduce2(`mx,`my,`r,null,null,`x,`y,`o)>);
+        case MapAggregateReduce2(`mx,`my,lambda(`v,`b),`acc,`zero,`x,`y,false):
+            if (!Config.flink_mode)
+                fail;
+            return #<MapAggregateReduce2(`mx,`my,lambda(`v,`b),`acc,`zero,
+                                         `(physical_plan(x)),
+                                         `(physical_plan(y)),false)>;
         // convert a reduce-side join to a fragment-replicate join, if either of the join
         //   inputs is small to fit in memory (dataset sizes are extracted from file statistics)
         case MapAggregateReduce2(`mx,`my,
@@ -750,6 +756,8 @@ final public class PlanGeneration extends AlgebraicOptimization {
         case CrossProduct(`mx,`my,`r,`x,`y):
             return physical_plan(#<CrossAggregateProduct(`mx,`my,`r,null,null,`x,`y)>);
         case CrossAggregateProduct(`mx,`my,lambda(`v,`b),`acc,`zero,`x,`y):
+            if (Config.flink_mode)
+                fail;
             Tree X = new_var();
             Tree Y = new_var();
             Tree nv = new_var();

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/SystemFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/SystemFunctions.java b/core/src/main/java/org/apache/mrql/SystemFunctions.java
index 5c0ed4b..b7385fb 100644
--- a/core/src/main/java/org/apache/mrql/SystemFunctions.java
+++ b/core/src/main/java/org/apache/mrql/SystemFunctions.java
@@ -361,10 +361,23 @@ final public class SystemFunctions {
     }
  
     /** used in avg */
-    public static MR_double avg_value ( MRData t ) {
-        MR_double sum = (MR_double)((Tuple)t).first();
-        MR_long count = (MR_long)((Tuple)t).second();
-        return new MR_double(sum.get()/count.get());
+    public static MR_double avg_value ( MRData value ) {
+        Tuple t = (Tuple)value;
+        double sum = 0.0;
+        if (t.first() instanceof MR_double)
+            sum = ((MR_double)t.first()).get();
+        else if (t.first() instanceof MR_int)
+            sum = ((MR_int)t.first()).get();
+        else if (t.first() instanceof MR_long)
+            sum = ((MR_long)t.first()).get();
+        else if (t.first() instanceof MR_float)
+            sum = ((MR_float)t.first()).get();
+        long count = 0;
+        if (t.second() instanceof MR_long)
+            count = ((MR_long)t.second()).get();
+        else if (t.second() instanceof MR_int)
+            count = ((MR_int)t.second()).get();
+        return new MR_double(sum/count);
     }
 
     public static MR_string text ( Union node ) {

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/Tuple.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/Tuple.java b/core/src/main/java/org/apache/mrql/Tuple.java
index bced3dd..13b0eeb 100644
--- a/core/src/main/java/org/apache/mrql/Tuple.java
+++ b/core/src/main/java/org/apache/mrql/Tuple.java
@@ -28,6 +28,8 @@ final public class Tuple extends MRData {
 
     MRData[] tuple;
 
+    public Tuple () { tuple = new MRData[0]; }
+
     public Tuple ( int size ) {
         tuple = new MRData[size];
     }

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/Union.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/Union.java b/core/src/main/java/org/apache/mrql/Union.java
index 540b6d9..5d92f32 100644
--- a/core/src/main/java/org/apache/mrql/Union.java
+++ b/core/src/main/java/org/apache/mrql/Union.java
@@ -29,6 +29,8 @@ public class Union extends MRData {
     private byte tag;
     private MRData value;
 
+    public Union () {}
+
     public Union ( byte tag, MRData value ) {
         this.tag = tag;
         this.value = value;

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/XMLParser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/XMLParser.java b/core/src/main/java/org/apache/mrql/XMLParser.java
index a25acbd..f55faba 100644
--- a/core/src/main/java/org/apache/mrql/XMLParser.java
+++ b/core/src/main/java/org/apache/mrql/XMLParser.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.conf.Configuration;
 
 
 /** An XML parser */
-final public class XMLParser implements Parser {
+public class XMLParser implements Parser {
     String[] tags;          // split tags
     Tree xpath;             // XPath query for fragmentation
     XMLSplitter splitter;

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/test/java/org/apache/mrql/QueryTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/mrql/QueryTest.java b/core/src/test/java/org/apache/mrql/QueryTest.java
index b0257dd..a675c9d 100644
--- a/core/src/test/java/org/apache/mrql/QueryTest.java
+++ b/core/src/test/java/org/apache/mrql/QueryTest.java
@@ -49,7 +49,7 @@ public abstract class QueryTest extends TestCase {
 		resultDir = new File(TEST_RESULT_DIR);
 		resultDir.mkdirs();
 
-		if(evaluator==null)
+		// if(evaluator==null) // the spark evaluator needs to be recreated
 			evaluator = createEvaluator();
 		Translator.global_reset();
                 for ( Enumeration en = LogManager.getCurrentLoggers(); en.hasMoreElements(); )

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/pom.xml
----------------------------------------------------------------------
diff --git a/flink/pom.xml b/flink/pom.xml
new file mode 100644
index 0000000..a76cda7
--- /dev/null
+++ b/flink/pom.xml
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.mrql</groupId>
+  <artifactId>mrql-flink</artifactId>
+  <packaging>jar</packaging>
+  <name>Apache MRQL Flink mode</name>
+  <description>Apache MRQL evaluation in Flink mode on Apache Flink</description>
+  <url>http://mrql.incubator.apache.org/</url>
+  <inceptionYear>2013</inceptionYear>
+
+  <parent>
+    <groupId>org.apache.mrql</groupId>
+    <artifactId>mrql-parent</artifactId>
+    <version>0.9.4-incubating-SNAPSHOT</version>
+  </parent>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.mrql</groupId>
+      <artifactId>mrql-gen</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.mrql</groupId>
+      <artifactId>mrql-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.flink</groupId>
+      <artifactId>flink-java</artifactId>
+      <version>${flink.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.flink</groupId>
+      <artifactId>flink-clients</artifactId>
+      <version>${flink.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.mrql</groupId>
+      <artifactId>mrql-core</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <version>1.8</version>
+        <executions>
+          <execution>
+            <phase>generate-sources</phase>
+            <goals><goal>add-source</goal></goals>
+            <configuration>
+              <sources>
+                <source>${project.build.directory}/generated-sources</source>
+              </sources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+	<groupId>org.apache.maven.plugins</groupId>
+	<artifactId>maven-shade-plugin</artifactId>
+	<version>2.1</version>
+	<executions>
+	  <execution>
+	    <phase>package</phase>
+	    <goals>
+	      <goal>shade</goal>
+	    </goals>
+	    <configuration>
+             <artifactSet>
+	       <includes>
+		 <include>org.apache.mrql:*</include>
+	       </includes>
+	       <excludes>
+		 <exclude>org.apache.mrql:mrql-gen</exclude>
+                 <exclude>org.apache.mrql:mrql-core</exclude>
+	       </excludes>
+              </artifactSet>
+	    </configuration>
+	  </execution>
+	</executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <version>2.4</version>
+	<configuration>
+	  <finalName>mrql-flink-${project.version}</finalName>
+	  <outputDirectory>${project.parent.basedir}/lib</outputDirectory>
+	</configuration>
+      </plugin>
+      <plugin>
+	<artifactId>maven-antrun-plugin</artifactId>
+	<version>1.7</version>
+	<executions>
+	  <execution>
+	    <phase>generate-sources</phase>
+	    <goals>
+	      <goal>run</goal>
+	    </goals>
+	    <configuration>
+              <target>
+                <mkdir dir="${project.build.directory}/generated-sources/org/apache/mrql" />
+		<property name="compile_classpath" refid="maven.compile.classpath" />
+		<fileset id="mr.gen.path" dir="src/main/java/org/apache/mrql" includes="*.gen" />
+		<pathconvert pathsep=" " property="mr.gen.files" refid="mr.gen.path" />
+		<java classname="org.apache.mrql.gen.Main" classpath="../lib/mrql-gen-${project.version}.jar:${compile_classpath}">
+		  <arg line="${mr.gen.files}" />
+		  <arg line="-o" />
+		  <arg file="${project.build.directory}/generated-sources/org/apache/mrql" />
+		</java>
+	      </target>
+  	    </configuration>
+	  </execution>
+	</executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FData.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FData.java b/flink/src/main/java/org/apache/mrql/FData.java
new file mode 100644
index 0000000..7cb288c
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FData.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mrql;
+
+import java.io.IOException;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.types.CopyableValue;
+
+
+final public class FData implements CopyableValue<FData>, Comparable<FData> {
+    MRData data;
+
+    public FData () {}
+
+    public FData ( MRData d ) { data = d; }
+
+    public MRData data () { return data; }
+
+    @Override
+    public void read ( DataInputView in ) throws IOException {
+        data = MRContainer.read(in);
+    }
+
+    @Override
+    public void write ( DataOutputView out ) throws IOException {
+        data.write(out);
+    }
+
+    @Override
+    public int getBinaryLength () { return -1; }
+
+    @Override
+    public void copyTo ( FData target ) {
+        target.data = data;
+    }
+
+    @Override
+    public void copy ( DataInputView source, DataOutputView target ) throws IOException {
+        data = MRContainer.read(source);
+        data.write(target);
+    }
+
+    @Override
+    public int compareTo ( FData x ) { return data.compareTo(x.data); }
+
+    @Override
+    public boolean equals ( Object x ) {
+        return x instanceof FData && data.equals(((FData)x).data);
+    }
+
+    @Override
+    public int hashCode () { return data.hashCode(); }
+
+    @Override
+    public String toString () { return data.toString(); }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FDataContainer.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FDataContainer.java b/flink/src/main/java/org/apache/mrql/FDataContainer.java
new file mode 100644
index 0000000..07eaa22
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FDataContainer.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mrql;
+
+import java.io.IOException;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.core.io.IOReadableWritable;
+
+final public class FDataContainer extends MRContainer implements IOReadableWritable, Comparable<MRContainer> {
+
+    FDataContainer ( MRData d ) { super(d); }
+
+    @Override
+    public void read ( DataInputView in ) throws IOException {
+        data = super.read(in);
+    }
+
+    @Override
+    public void write ( DataOutputView out ) throws IOException {
+        super.write(out);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkBinaryInputFormat.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkBinaryInputFormat.java b/flink/src/main/java/org/apache/mrql/FlinkBinaryInputFormat.java
new file mode 100644
index 0000000..5969c1e
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkBinaryInputFormat.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.*;
+import org.apache.flink.api.common.io.BinaryInputFormat;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.api.common.io.FileInputFormat;
+
+
+/** Input format for sequence files */
+final public class FlinkBinaryInputFormat extends FlinkMRQLFileInputFormat {
+    public FlinkBinaryInputFormat () {}
+
+    public static final class FDataInputFormat extends BinaryInputFormat<FData> {
+        @Override
+        protected FData deserialize ( FData reuse, DataInputView in ) throws IOException {
+            reuse.read(in);
+            return reuse;
+        }
+    }
+
+    /** the Flink input format for this input */
+    public FileInputFormat<FData> inputFormat ( String path  ) {
+        FDataInputFormat sf = new FDataInputFormat();
+        sf.setFilePath(path.toString());
+        return sf;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkDataSource.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkDataSource.java b/flink/src/main/java/org/apache/mrql/FlinkDataSource.java
new file mode 100644
index 0000000..6c98ade
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkDataSource.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.mrql.gen.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.io.IOException;
+import java.io.Serializable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.common.accumulators.Accumulator;
+import org.apache.flink.api.common.accumulators.LongCounter;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.Collector;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.core.fs.FileInputSplit;
+
+
+/** A data source for storing a Flink result (a Flink DataSet ) */
+final public class FlinkDataSource extends DataSource implements Serializable {
+    public DataSet<FData> data_set;
+
+    /** Used in MRQL total aggregations in FlinkEvaluator.aggregate */
+    public static final class Reducer implements Accumulator<FData,String> {
+        public FData accumulated_value;
+        public FData zero;
+        public String acc_fnc;
+        public String merge_fnc;
+        transient Function acc;
+        transient Function merge;
+
+        public Reducer () {}
+
+        public Reducer ( MRData zero, String acc_fnc, String merge_fnc ) {
+            this.zero = new FData(zero);
+            accumulated_value = this.zero;
+            this.acc_fnc = acc_fnc;
+            this.merge_fnc = merge_fnc;
+        }
+
+        public void add ( FData value ) {
+            if (acc == null)
+                try {
+                    acc = Interpreter.evalF(Tree.parse(acc_fnc),null);
+                } catch (Exception ex) {
+                    throw new Error(ex);
+                };
+            accumulated_value = new FData(acc.eval(new Tuple(accumulated_value.data(),value.data())));
+        }
+
+        public String getLocalValue () {
+            // due to a Flink bug, we cannot return a custom object (FData) from an accumulator
+            return accumulated_value.data().toString();
+        }
+
+        public void resetLocal () {
+            accumulated_value = zero;
+        }
+
+        public void merge ( Accumulator<FData,String> other ) {
+            if (merge == null)
+                try {
+                    merge = Interpreter.evalF(Tree.parse(merge_fnc),null);
+                } catch (Exception ex) {
+                    throw new Error(ex);
+                };
+            accumulated_value = new FData(merge.eval(new Tuple(accumulated_value.data(),
+                                                               ((Reducer)other).accumulated_value.data())));
+        }
+
+        public void write ( DataOutputView out ) throws IOException {
+            accumulated_value.write(out);
+            zero.write(out);
+            out.writeUTF(acc_fnc);
+            out.writeUTF(merge_fnc);
+        }
+
+        public void read ( DataInputView in ) throws IOException {
+            accumulated_value = new FData();
+            zero = new FData();
+            accumulated_value.read(in);
+            zero.read(in);
+            acc_fnc = in.readUTF();
+            merge_fnc = in.readUTF();
+        }
+    }
+
+    FlinkDataSource ( DataSet<FData> d, String path, boolean to_be_merged  ) {
+        super();
+        this.data_set = d;
+        this.path = path;
+        this.inputFormat = FlinkBinaryInputFormat.class;
+        this.to_be_merged = to_be_merged;
+    }
+
+    public static final class size_mapper extends RichFlatMapFunction<FData,FData> {
+        private LongCounter c = new LongCounter();
+
+        @Override
+        public void open ( org.apache.flink.configuration.Configuration parameters ) throws Exception {
+            super.open(parameters);
+            getRuntimeContext().addAccumulator("count",c);
+        }
+
+        @Override
+        public void flatMap ( FData value, Collector<FData> out ) throws Exception {
+            c.add(1L);
+        }
+    }
+
+    /** not used */
+    @Override
+    public long size ( Configuration conf ) {
+        return Long.MAX_VALUE;
+    }
+
+    /** return the first num values */
+    @Override
+    public List<MRData> take ( int num ) {
+        try {
+            // used for printing query results; currently, terribly inefficient
+            List<MRData> a = new ArrayList<MRData>();
+            final FData container = new FData();
+            if (!to_be_merged) {
+                FileInputFormat<FData> sf = (new FlinkBinaryInputFormat()).inputFormat(path);
+                FileInputSplit[] splits = sf.createInputSplits(1);
+                long i = 0;
+                for ( FileInputSplit split: splits ) {
+                    sf.open(split);
+                    while (!sf.reachedEnd() && (num < 0 || i < num)) {
+                        a.add(sf.nextRecord(container).data());
+                        i++;
+                    };
+                    if (num > 0 && i >= num)
+                        break;
+                };
+                sf.close();
+                return a;
+            } else { // needs merging since splits came out of sorting
+                FlinkBinaryInputFormat ifbif = new FlinkBinaryInputFormat();
+                FileInputFormat<FData> sf = ifbif.inputFormat(path);
+                FileInputSplit[] splits = sf.createInputSplits(1);
+                FileInputFormat<FData>[] s = new FlinkBinaryInputFormat.FDataInputFormat[splits.length];
+                MRData[] key = new MRData[splits.length];
+                MRData[] value = new MRData[splits.length];
+                int min = 0;
+                int i = 0;
+                for ( FileInputSplit split: splits ) {
+                    s[i] = ifbif.inputFormat(path);
+                    s[i].open(split);
+                    i++;
+                };
+                for ( int j = 0; j < splits.length; j++ ) {
+                    if (!s[j].reachedEnd()) {
+                        Tuple v = (Tuple)s[j].nextRecord(container).data();
+                        key[j] = v.first();
+                        value[j] = v.second();
+                    }
+                };
+                long c = 0;
+                while ( (num < 0 || c < num) && min >= 0) {
+                    min = -1;
+                    for ( int j = 0; j < splits.length; j++ )
+                        if (key[j] != null && (min < 0 || key[j].compareTo(key[min]) < 0))
+                            min = j;
+                    if (min >= 0) {
+                        a.add(value[min]);
+                        if (!s[min].reachedEnd()) {
+                            Tuple v = (Tuple)s[min].nextRecord(container).data();
+                            key[min] = v.first();
+                            value[min] = v.second();
+                        } else key[min] = null;
+                    };
+                    c++;
+                };
+                for ( int j = 0; j < splits.length; j++ )
+                    s[j].close();
+                return a;
+            }
+        } catch (IOException ex) {
+            throw new Error(ex);
+        }
+    }
+
+    /** merge the splits of a flink data source that have already been sorted */
+    public Bag merge () {
+        try {
+            FlinkBinaryInputFormat ifbif = new FlinkBinaryInputFormat();
+            FileInputFormat<FData> sf = ifbif.inputFormat(path);
+            FileInputSplit[] splits = sf.createInputSplits(1);
+            final int n = splits.length;
+            final FileInputFormat<FData>[] s = new FlinkBinaryInputFormat.FDataInputFormat[n];
+            int i = 0;
+            for ( FileInputSplit split: splits ) {
+                s[i] = ifbif.inputFormat(path);
+                s[i].open(split);
+                i++;
+            };
+            return new Bag(new BagIterator () {
+                    int min = 0;
+                    boolean first = true;
+                    MRData[] keys = new MRData[n];
+                    MRData[] values = new MRData[n];
+                    MRData key, value;
+                    final FData container = new FData();
+                    public boolean hasNext () {
+                        try {
+                            if (first) {
+                                first = false;
+                                for ( int i = 0; i < n; i++ ) {
+                                    if (!s[i].reachedEnd()) {
+                                        Tuple v = (Tuple)s[i].nextRecord(container).data();
+                                        keys[i] = v.first();
+                                        values[i] = v.second();
+                                    }
+                                }
+                            };
+                            min = -1;
+                            for ( int i = 0; i < n; i++ )
+                                if (keys[i] != null && (min < 0 || keys[i].compareTo(keys[min]) < 0))
+                                    min = i;
+                            if (min >= 0) {
+                                key = keys[min];
+                                value = values[min];
+                                if (!s[min].reachedEnd()) {
+                                    Tuple v = (Tuple)s[min].nextRecord(container).data();
+                                    keys[min] = v.first();
+                                    values[min] = v.second();
+                                } else keys[min] = null;
+                                return true;
+                            } else {
+                                for ( int i = 0; i < n; i++ )
+                                    s[i].close();
+                                return false;
+                            }
+                        } catch (IOException e) {
+                            throw new Error("Cannot merge values from an intermediate result: "+e);
+                        }
+                    }
+                    public MRData next () { return value; }
+                });
+        } catch (IOException e) {
+            throw new Error("Cannot merge values from an intermediate result: "+e);
+        }
+    }
+
+    /** Used in MRQL total aggregations in FlinkEvaluator.aggregate */
+    public static final class reduce_mapper extends RichFlatMapFunction<FData,FData> {
+        Reducer r;
+
+        reduce_mapper ( MRData zero, String acc, String merge ) {
+            r = new Reducer(zero,acc,merge);
+        }
+
+        @Override
+        public void open ( org.apache.flink.configuration.Configuration parameters ) throws Exception {
+            super.open(parameters);
+            getRuntimeContext().addAccumulator("reducer",r);
+        }
+
+        @Override
+        public void flatMap ( FData value, Collector<FData> out ) throws Exception {
+            r.add(value);
+        }
+    }
+
+    /** accumulate all dataset values */
+    @Override
+    public MRData reduce ( MRData zero, Function acc ) {
+        throw new Error("Flink reduce operation is not implemented yet");
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen b/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
new file mode 100644
index 0000000..140a7f2
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
@@ -0,0 +1,691 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.io.PrintStream;
+import java.net.URI;
+import java.net.URL;
+import org.apache.mrql.gen.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.flink.util.Collector;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.RemoteEnvironment;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.IterativeDataSet;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.io.LocalCollectionOutputFormat;
+import org.apache.flink.api.java.functions.RichMapFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichCoGroupFunction;
+import org.apache.flink.api.java.functions.RichCrossFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
+import org.apache.flink.api.java.functions.RichFilterFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.GroupReduceOperator;
+
+
+/** Evaluates physical plans in Apache Flink mode */
+final public class FlinkEvaluator extends Evaluator {
+    final static URL flink_jar = FlinkEvaluator.class.getProtectionDomain().getCodeSource().getLocation();
+    public static ExecutionEnvironment flink_env;
+
+    /** initialize the Flink evaluator */
+    final public void init ( Configuration conf ) {
+        DataSource.parserDirectory = new DataSource.ParserDirectory();
+        DataSource.parserDirectory.put("xml",FlinkXMLParser.class);
+        DataSource.parserDirectory.put("json",FlinkJsonParser.class);
+        DataSource.parserDirectory.put("line",FlinkLineParser.class);
+        if (Config.local_mode) {
+            flink_env = ExecutionEnvironment.createLocalEnvironment(Config.nodes);
+            // curently, the compiler desn't work in local mode
+            Config.compile_functional_arguments = false;
+            Config.write(Plan.conf);
+        } else flink_env = ExecutionEnvironment.getExecutionEnvironment();
+    }
+
+    /** shutdown the Flink evaluator */
+    final public void shutdown ( Configuration conf ) {
+    }
+
+    final public void initialize_query () {
+        try {
+            if (!Config.local_mode) {
+                Plan.distribute_compiled_arguments(Plan.conf);
+                String master_node = System.getenv("FLINK_MASTER");
+                if (Config.compile_functional_arguments)
+                    flink_env = ExecutionEnvironment.createRemoteEnvironment(master_node,6123,
+                                        flink_jar.toURI().getPath(),Plan.conf.get("mrql.jar.path"));
+                else flink_env = ExecutionEnvironment.createRemoteEnvironment(master_node,6123,
+                                        flink_jar.toURI().getPath());
+                flink_env.setDefaultLocalParallelism(Config.nodes);
+            };
+        } catch (Exception ex) {
+            throw new Error(ex);
+        }
+    }
+
+    final public Configuration new_configuration () {
+        return new Configuration();
+    }
+
+    /** return the FileInputFormat for parsed files (CSV, XML, JSON, etc) */
+    final public Class<? extends MRQLFileInputFormat> parsedInputFormat () {
+        return FlinkParsedInputFormat.class;
+    }
+
+    /** return the FileInputFormat for binary files */
+    final public Class<? extends MRQLFileInputFormat> binaryInputFormat () {
+        return FlinkBinaryInputFormat.class;
+    }
+
+    /** return the FileInputFormat for data generator files */
+    final public Class<? extends MRQLFileInputFormat> generatorInputFormat () {
+        return FlinkGeneratorInputFormat.class;
+    }
+
+    /** returns the absolute path relative to the directory that contains the MRQL executable */
+    private String absolute_path ( String path) {
+        try {
+            URI uri = new URI(path);
+            if (uri.getScheme() == null)
+                if (uri.toString().startsWith("/"))
+                    uri = new URI("file:"+uri);
+                else uri = new URI("file:"+System.getProperty("user.dir")+"/"+uri);
+            uri = uri.normalize();
+            return uri.toString();
+        } catch (Exception ex) {
+            throw new Error("Wrong pathname: "+path);
+        }
+    }
+
+    /** dump MRQL data into a Flink binary file instead of a Hadoop Sequence file */
+    @Override
+    public void dump ( String file, Tree type, MRData data ) throws Exception {
+        Path path = new Path(absolute_path(file));
+        FileSystem fs = path.getFileSystem();
+        PrintStream ftp = new PrintStream(fs.create(path.suffix(".type"),true));
+        ftp.print("2@"+type.toString()+"\n");
+        ftp.close();
+        if (data instanceof MR_dataset)
+            data = Plan.collect(((MR_dataset)data).dataset());
+        DataSet<Long> d = flink_env.generateSequence(0L,0L); // need just one record
+        DataSet<FData> res = (data instanceof Bag)
+                             ? d.flatMap(new bag_dumper(new FData(data)))
+                             : d.map(new value_dumper(new FData(data)));
+        res.write(new FlinkOutputFormat(),path.toString(),
+                  org.apache.flink.core.fs.FileSystem.WriteMode.OVERWRITE);
+        flink_env.execute("MRQL dump");
+    }
+
+    public static final class bag_dumper extends RichFlatMapFunction<Long,FData> {
+        final FData container = new FData(new MR_int(0));
+        final FData value;
+
+        bag_dumper ( FData value ) { this.value = value; }
+
+        @Override
+        public void flatMap ( Long ignore, Collector<FData> out ) throws Exception {
+            for ( MRData e: (Bag)value.data() ) {
+                container.data = e;
+                out.collect(container);
+            }
+        }
+    }
+
+    public static final class value_dumper extends RichMapFunction<Long,FData> {
+        final FData value;
+
+        value_dumper ( FData value ) { this.value = value; }
+
+        @Override
+        public FData map ( Long ignore ) {
+            return value;
+        }
+    }
+
+    private static MRData aggregate ( DataSet<FData> data_set, MRData zero, Tree acc_fnc, Tree merge_fnc ) {
+        DataSet<FData> d = data_set.flatMap(new FlinkDataSource.reduce_mapper(zero,acc_fnc.toString(),
+                                                                              merge_fnc.toString()));
+        d.print();  // needs a datasink (prints the aggregation value)
+        try {
+            // due to a Flink bug, we cannot return a custom object (FData) from an accumulator
+            String value = FlinkEvaluator.flink_env.execute("MRQL aggregator").getAccumulatorResult("reducer");
+            return MRQL.query(value);  // parse the value using the MRQL query parser (ugly)
+        } catch (Exception ex) {
+            throw new Error("Evaluation error during aggregation: "+ex);
+        }
+    }
+
+    /** The Aggregate physical operator
+     * @param merge  the accumulator function from (T,T) to T
+     * @param zero  the zero element of type T
+     * @param plan the plan that constructs the dataset that contains the bag of values {T}
+     * @param env contains bindings fro variables to values (MRData)
+     * @return the aggregation result of type T
+     */
+    final public MRData aggregate ( Tree merge,
+                                    Tree zero,
+                                    Tree plan,
+                                    Environment env ) throws Exception {
+        MRData z = evalE(zero,env);
+        match plan {
+        case AggregateMap(`m,`acc,_,`s):
+            return aggregate(evalD(#<cMap(`m,`s)>,env),z,acc,merge);
+        case MapAggregateReduce(`m,`r,`acc,_,`s,`o):
+            if (acc.equals(#<null>))
+                fail;
+            return aggregate(evalD(#<MapReduce(`m,`r,`s,`o)>,env),z,acc,merge);
+        case CrossAggregateProduct(`mx,`my,`r,`acc,_,`x,`y):
+            if (acc.equals(#<null>))
+                fail;
+            return aggregate(evalD(#<CrossProduct(`mx,`my,`r,`x,`y)>,env),z,acc,merge);
+        case MapAggregateReduce2(`mx,`my,`r,`acc,_,`x,`y,`o):
+            if (acc.equals(#<null>))
+                fail;
+            return aggregate(evalD(#<MapReduce2(`mx,`my,`r,`x,`y,`o)>,env),z,acc,merge);
+        case MapAggregateJoin(`mx,`my,`r,`acc,_,`x,`y):
+            if (acc.equals(#<null>))
+                fail;
+            return aggregate(evalD(#<MapJoin(`mx,`my,`r,`x,`y)>,env),z,acc,merge);
+        };
+        throw new Error("Unrecognized aggregation: "+plan);
+    }
+
+    /** Evaluate a loop a fixed number of times. Doesn't use Flink iterations */
+    final public Tuple loop ( Tree e, Environment env ) throws Exception {
+        match e {
+        case Loop(lambda(tuple(...vs),tuple(...bs)),tuple(...ss),`num):
+            final int limit = ((MR_int)evalE(num,env)).get();
+            MR_flink[] s = new MR_flink[vs.length()];
+            String[] path = new String[vs.length()];
+            for ( int i = 0; i < vs.length(); i++ )
+                s[i] = new MR_flink(eval(ss.nth(i),env));
+            for ( int n = 0; n < limit; n++ ) {
+                Environment nenv = env;
+                for ( int i = 0; i < vs.length(); i ++ )
+                    nenv = new Environment(vs.nth(i).toString(),s[i],nenv);
+                for ( int i = 0; i < vs.length(); i ++ )
+                    s[i] = new MR_flink(eval(bs.nth(i),nenv));
+            };
+            for ( int i = 0; i < vs.length(); i++ ) {
+                path[i] = absolute_path(Plan.new_path(Plan.conf));
+                s[i].flink().write(new FlinkOutputFormat(),path[i]);
+            };
+            flink_env.execute("MRQL loop");
+            final FlinkBinaryInputFormat bif = new FlinkBinaryInputFormat();
+            Tuple t = new Tuple(vs.length());
+            for ( int i = 0; i < vs.length(); i++ )
+                t.set(i,bif.materialize(new org.apache.hadoop.fs.Path(path[i])));
+            return t;
+        };
+        throw new Error("Wrong Loop format: "+e);
+    }
+
+    private static boolean needs_sorting ( Tree e ) {
+        match e {
+        case MapReduce(`m,`r,`s,true): return true;
+        case MapCombineReduce(`m,`c,`r,`s,true): return true;
+        };
+        return false;
+    }
+
+    /** Evaluate an MRQL physical plan and print tracing info
+     * @param e the physical plan
+     * @param env contains bindings fro variables to values (MRData)
+     * @return an MRQL DataSet (stored in HDFS)
+     */
+    final public org.apache.mrql.DataSet eval ( final Tree e,
+                                                final Environment env,
+                                                final String counter ) {
+        DataSet<FData> d = eval(e,env);
+        try {
+            String path = absolute_path(Plan.new_path(Plan.conf));
+            d.write(new FlinkOutputFormat(),path);
+            flink_env.execute("MRQL query");
+            return new org.apache.mrql.DataSet(new FlinkDataSource(d,path,needs_sorting(e)),0L,0L);
+        } catch (Exception ex) {
+            throw new Error("Cannot evaluate the query: "+e);
+        }
+    }
+
+    /** Evaluate an MRQL physical plan using Flink and print tracing info
+     * @param e the physical plan
+     * @param env contains bindings from variables to values (MRData)
+     * @return a Flink DataSet
+     */
+    final public DataSet<FData> eval ( final Tree e, final Environment env ) {
+        if (Config.trace_execution) {
+            tab_count += 3;
+            System.out.println(tabs(tab_count)+print_query(e));
+        };
+        final DataSet<FData> res = evalD(e,env);
+        if (Config.trace_execution) 
+            try {
+                List<FData> out = new ArrayList<FData>();
+                res.output(new LocalCollectionOutputFormat(out));
+                System.out.println(tabs(tab_count)+"-> "+out);
+                tab_count -= 3;
+            } catch (Exception ex) {
+                throw new Error("Cannot collect the operator output: "+e);
+            };
+        return res;
+    }
+
+    /* convert an MRQL lambda to a Flink Function */
+    private static RichFlatMapFunction<FData,FData> cmap_fnc ( final Tree fnc ) {
+        final Function f = evalF(fnc,null);
+        return new RichFlatMapFunction<FData,FData>() {
+            @Override
+            public void flatMap ( FData value, Collector<FData> out ) throws Exception {
+                for ( MRData e: (Bag)f.eval(value.data()) )
+                    out.collect(new FData(e));
+            }
+        };
+    }
+
+    /* convert the stream of pairs to a bag of their second element */
+    public static Bag bag ( Iterable<FData> values ) {
+        Bag s = new Bag();
+        for ( FData value: values )
+            s.add(((Tuple)value.data()).second());
+        return s;
+    }
+
+    /* group-by s and then reduce by reduce_fnc (optional: use combine_fnc) */
+    private static DataSet<FData> groupBy ( DataSet<FData> s, Tree combine_fnc, Tree reduce_fnc ) {
+        final Function combiner = (combine_fnc == null) ? null : evalF(combine_fnc,null);
+        final Function reducer = evalF(reduce_fnc,null);
+        GroupReduceOperator<FData,FData> ro = s.groupBy(new KeySelector<FData,FData>() {
+                @Override
+                public FData getKey ( FData value ) {
+                    return new FData(((Tuple)value.data()).first());
+                }
+            }).reduceGroup(new RichGroupReduceFunction<FData,FData>() {
+                @Override
+                public void reduce ( final Iterable<FData> values, Collector<FData> out ) {
+                    Bag s = new Bag();
+                    MRData key = null;
+                    for ( FData value: values ) {
+                        Tuple t = (Tuple)value.data();
+                        key = t.first();
+                        s.add(t.second());
+                    };
+                    for ( MRData v: (Bag)reducer.eval(new Tuple(key,s)) )
+                        out.collect(new FData(v));
+                }
+                @Override
+                public void combine ( Iterable<FData> values, Collector<FData> out ) throws Exception {
+                    Bag c = null;
+                    MRData key = null;
+                    for ( FData v: values )
+                        if (c == null) {
+                            c = new Bag(((Tuple)v.data()).second());
+                            key = ((Tuple)v.data()).first();
+                        } else
+                            c = (Bag)combiner.eval(new Tuple(key,c.add_element(((Tuple)v.data()).second())));
+                    for ( MRData x: c )
+                        out.collect(new FData(new Tuple(key,x)));
+                }
+            });
+        ro.setCombinable(combiner != null);
+        return ro;
+    }
+
+    /* group-by and sort s and then reduce by reduce_fnc (optional: use combine_fnc) */
+    private static DataSet<FData> sortBy ( DataSet<FData> s, Tree combine_fnc, Tree reduce_fnc ) {
+        final Function combiner = (combine_fnc == null) ? null : evalF(combine_fnc,null);
+        final Function reducer = evalF(reduce_fnc,null);
+        GroupReduceOperator<Tuple2<FData,FData>,FData> ro
+            = s.map(new RichMapFunction<FData,Tuple2<FData,FData>>() {
+                    @Override
+                    public Tuple2<FData,FData> map ( FData value ) {
+                        Tuple t = (Tuple)value.data();
+                        return new Tuple2<FData,FData>(new FData(t.first()),new FData(t.second()));
+                    }
+                }).groupBy(0).sortGroup(0,Order.ASCENDING)
+              .reduceGroup(new RichGroupReduceFunction<Tuple2<FData,FData>,FData>() {
+                    @Override
+                    public void reduce ( final Iterable<Tuple2<FData,FData>> values, Collector<FData> out ) {
+                        Bag s = new Bag();
+                        MRData key = null;
+                        for ( Tuple2<FData,FData> value: values ) {
+                            key = value.f0.data();
+                            s.add(value.f1.data());
+                        };
+                        for ( MRData v: (Bag)reducer.eval(new Tuple(key,s)) )
+                            out.collect(new FData(new Tuple(key,v)));
+                    }
+                    @Override
+                    public void combine ( Iterable<Tuple2<FData,FData>> values,
+                                          Collector<Tuple2<FData,FData>> out ) throws Exception {
+                        Bag c = null;
+                        FData key = null;
+                        for ( Tuple2<FData,FData> v: values )
+                            if (c == null) {
+                                c = new Bag(v.f1.data());
+                                key = v.f0;
+                            } else
+                                c = (Bag)combiner.eval(new Tuple(key.data(),c.add_element(v.f0.data())));
+                        for ( MRData x: c )
+                            out.collect(new Tuple2<FData,FData>(key,new FData(x)));
+                    }
+                  });
+        ro.setCombinable(combiner != null);
+        return ro;
+    }
+
+    public static final class join_key implements KeySelector<FData,FData> {
+        @Override
+        public FData getKey ( FData value ) {
+            return new FData(((Tuple)value.data()).first());
+        }
+    }
+
+    public static final class join_reducer extends RichCoGroupFunction<FData,FData,FData> {
+        final Function f;
+
+        join_reducer ( Tree fnc ) {
+            f = evalF(fnc,null);
+        }
+
+        @Override
+        public void coGroup ( Iterable<FData> xs, Iterable<FData> ys, Collector<FData> out ) {
+            for ( MRData v: (Bag)f.eval(new Tuple(bag(xs),bag(ys))) )
+                out.collect(new FData(v));
+        }
+    }
+
+    public static final class cross_reducer extends RichCrossFunction<FData,FData,FData> {
+        final Function f;
+
+        cross_reducer ( Tree fnc ) {
+            f = evalF(fnc,null);
+        }
+
+        @Override
+        public FData cross ( FData x, FData y ) throws Exception {
+            return new FData(f.eval(new Tuple(x.data(),y.data())));
+        }
+    }
+
+    public static final class mapjoin_reducer extends RichJoinFunction<FData,FData,FData> {
+        final Function f;
+
+        mapjoin_reducer ( Tree fnc ) {
+            f = evalF(fnc,null);
+        }
+
+        @Override
+        public FData join ( FData x, FData y ) throws Exception {
+            return new FData(f.eval(new Tuple(x.data(),y.data())));
+        }
+    }
+
+    /** restore the global bindings at the client node */
+    public static final class restore_global_functions extends RichMapFunction<FData,FData> {
+        Environment client_env;
+
+        restore_global_functions () {
+            client_env = Interpreter.global_env;
+        }
+
+        @Override
+        public void open ( org.apache.flink.configuration.Configuration parameters ) throws Exception {
+            Interpreter.set_global_bindings(client_env);
+        }
+
+        @Override
+        public FData map ( FData value ) throws Exception { return value; }
+    }
+
+    public static final class generator_mapper extends RichMapFunction<Long,FData> {
+        @Override
+        public FData map ( Long value ) throws Exception {
+            return new FData(new MR_long(value));
+        }
+    }
+
+    public static final class parsed_mapper extends RichFlatMapFunction<String,FData> {
+        final Parser parser;
+
+        parsed_mapper ( Class<? extends Parser> parser_class ) throws Exception {
+            parser = parser_class.newInstance();
+        }
+
+        @Override
+        public void flatMap ( String value, Collector<FData> out ) throws Exception {
+            for ( MRData e: parser.parse(value) )
+                out.collect(new FData(e));
+        }
+    }
+
+    public static final class group_join_left extends RichFlatMapFunction<FData,FData> {
+        final MRData one = new MR_byte(1);
+        final Function fgx;
+        final int m, n;
+
+        group_join_left ( Tree gx, int m, int n ) {
+            this.m = m;
+            this.n = n;
+            fgx = evalF(gx,null);
+        }
+
+        @Override
+        public void flatMap ( FData value, Collector<FData> out ) throws Exception {
+            for ( int i = 0; i < n; i++ ) {
+                final MRData key = new MR_int((fgx.eval(value.data()).hashCode() % m)+m*i);
+                out.collect(new FData(new Tuple(key,new Tuple(one,value.data()))));
+            }
+        }
+    }
+
+    public static final class group_join_right extends RichFlatMapFunction<FData,FData> {
+        final MRData two = new MR_byte(2);
+        final Function fgy;
+        final int m, n;
+
+        group_join_right ( Tree gy, int m, int n ) {
+            this.m = m;
+            this.n = n;
+            fgy = evalF(gy,null);
+        }
+
+        @Override
+        public void flatMap ( FData value, Collector<FData> out ) throws Exception {
+            for ( int j = 0; j < m; j++ ) {
+                final MRData key = new MR_int((fgy.eval(value.data()).hashCode() % n)*m+j);
+                out.collect(new FData(new Tuple(key,new Tuple(two,value.data()))));
+            }
+        }
+    }
+
+    public static final class group_join_reducer extends RichGroupReduceFunction<FData,FData> {
+        final Function fkx, fky, fgx, fgy, fc, fr;
+        final MRData z;
+
+        group_join_reducer ( Tree kx, Tree ky, Tree gx, Tree gy, Tree c, Tree r, Tree zero ) {
+            fkx = evalF(kx,null); fky = evalF(ky,null); fgx = evalF(gx,null); fgy = evalF(gy,null);
+            fr = evalF(r,null); fc = evalF(c,null); z = evalE(zero,null);
+        }
+
+        @Override
+        public void reduce ( final Iterable<FData> values, Collector<FData> out ) {
+            Bag xb = new Bag();
+            Bag yb = new Bag();
+            for ( FData value: values ) {
+                final Tuple t = (Tuple)value.data();
+                final Tuple p = (Tuple)t.second();
+                if (((MR_byte)p.first()).get() == 1)
+                    xb.add(new Tuple(t.first(),p.second()));
+                else yb.add(new Tuple(t.first(),p.second()));
+            }
+            final Bag b = MapReduceAlgebra.mergeGroupByJoin(fkx,fky,fgx,fgy,fc,z,fr,xb,yb);
+            for ( MRData v: b )
+                out.collect(new FData(v));
+        };
+    }
+
+    public static final class repeat_values extends RichMapFunction<FData,FData> {
+        @Override
+        public FData map ( FData value ) throws Exception {
+            return new FData(((Tuple)value.data()).first());
+        }
+    }
+
+    public static final class repeat_filter extends RichFilterFunction<FData> {
+        @Override
+        public boolean filter ( FData value ) throws Exception {
+            return ((MR_bool)(((Tuple)value.data()).second())).get();
+        }
+    }
+
+    /** Evaluate a MRQL physical plan
+     * @param e the physical plan
+     * @param env contains bindings from variables to values (MRData)
+     * @return a Flink DataSet
+     */
+    final private DataSet<FData> evalD ( final Tree e, final Environment env ) {
+        try {
+            match e {
+            case MapAggregateReduce(`m,`r,null,_,`s,`o):
+                return evalD(#<MapReduce(`m,`r,`s,`o)>,env);
+            case CrossAggregateProduct(`mx,`my,`r,null,_,`x,`y):
+                return evalD(#<CrossProduct(`mx,`my,`r,`x,`y)>,env);
+            case MapAggregateReduce2(`mx,`my,`r,null,_,`x,`y,`o):
+                return evalD(#<MapReduce2(`mx,`my,`r,`x,`y,`o)>,env);
+            case MapAggregateJoin(`mx,`my,`r,null,_,`x,`y):
+                return evalD(#<MapJoin(`mx,`my,`r,`x,`y)>,env);
+            case cMap(`f,`s):
+                return eval(s,env).flatMap(cmap_fnc(f));
+            case MapReduce(`m,`r,`s,true):
+                return sortBy(eval(s,env).flatMap(cmap_fnc(m)),null,r);
+            case MapReduce(`m,`r,`s,`o):
+                return groupBy(eval(s,env).flatMap(cmap_fnc(m)),null,r);
+            case MapCombineReduce(`m,`c,`r,`s,true):
+                return sortBy(eval(s,env).flatMap(cmap_fnc(m)),c,r);
+            case MapCombineReduce(`m,`c,`r,`s,`o):
+                return groupBy(eval(s,env).flatMap(cmap_fnc(m)),c,r);
+            case MapCombineReduce2(`mx,`my,`c,`r,`x,`y,`o):
+                return evalD(#<MapReduce2(`mx,`my,`r,`x,`y,`o)>,env);
+            case MapReduce2(`mx,`my,`r,`x,`y,`o):
+                final DataSet<FData> xs = eval(x,env).flatMap(cmap_fnc(mx));
+                final DataSet<FData> ys = eval(y,env).flatMap(cmap_fnc(my));
+                final DataSet<FData> ds = xs.coGroup(ys)
+                    .where(new join_key())
+                    .equalTo(new join_key())
+                    .with(new join_reducer(r));
+                if (o.equals(#<true>))
+                    return sortBy(ds,null,#<lambda(x,bag(x))>);
+                else return ds;
+            case MapJoin(`mx,`my,`r,`x,`y):
+                final DataSet<FData> xs = eval(x,env).flatMap(cmap_fnc(mx));
+                final DataSet<FData> ys = eval(y,env).flatMap(cmap_fnc(my));
+                return xs.joinWithTiny(ys)
+                    .where(new join_key())
+                    .equalTo(new join_key())
+                    .with(new mapjoin_reducer(r));
+            case CrossProduct(`mx,`my,`r,`x,`y):
+                final DataSet<FData> xs = eval(x,env).flatMap(cmap_fnc(mx));
+                final DataSet<FData> ys = eval(y,env).flatMap(cmap_fnc(my));
+                return xs.cross(ys).with(new cross_reducer(r))
+                         .flatMap(cmap_fnc(#<lambda(x,x)>));
+            case GroupByJoin(`kx,`ky,`gx,`gy,`acc,`zero,`r,`x,`y,`o):
+                final int n = Math.max(2,(int)Math.floor(Math.sqrt(Config.nodes)));
+                final int m = n;
+                // System.err.println("Using a groupBy join on a "+n+"*"+m+" grid of partitions");
+                final DataSet<FData> xs = eval(x,env).flatMap(new group_join_left(gx,m,n));
+                final DataSet<FData> ys = eval(y,env).flatMap(new group_join_right(gx,m,n));
+                return xs.union(ys).groupBy(new join_key())
+                         .reduceGroup(new group_join_reducer(kx,ky,gx,gy,acc,r,zero));
+            case Merge(`x,`y):
+                return eval(x,env).union(eval(y,env));
+            case Generator(`min,`max,`size):
+                final long from = ((MR_long)evalE(min,null)).get();
+                final long to = ((MR_long)evalE(max,null)).get();
+                return flink_env.generateSequence(from,to).map(new generator_mapper())
+                                .map(new restore_global_functions());
+            case BinarySource(`file,_):
+                final String path = absolute_path(((MR_string)evalE(file,null)).get());
+                new BinaryDataSource(path,Plan.conf);
+                final FileInputFormat<FData> sf = new FlinkBinaryInputFormat().inputFormat(path);
+                return flink_env.createInput(sf).map(new restore_global_functions());
+            case ParsedSource(`parser,`file,...args):
+                final String path = absolute_path(((MR_string)evalE(file,null)).get());
+                final Class<? extends Parser> parser_class = DataSource.parserDirectory.get(parser.toString());
+                if (parser_class == null)
+                    throw new Error("Unknown parser: "+parser);
+                final DataSource ds = new FlinkParsedDataSource(path,parser_class,args);
+                return flink_env.readFile(new FlinkParsedInputFormat.ParsedInputFormat(path),path)
+                                .map(new restore_global_functions());
+            case Repeat(lambda(`v,`b),`s,`n):
+                final int max_num = ((MR_int)evalE(n,null)).get();
+                final IterativeDataSet<FData> startOfIteration = eval(s,env).iterate(max_num);
+                final DataSet<FData> step = eval(b,new Environment(v.toString(),new MR_flink(startOfIteration),env));
+                final DataSet<FData> toBeFedBack = step.map(new repeat_values());
+                final DataSet<FData> terminationCriterion = step.filter(new repeat_filter());
+                return startOfIteration.closeWith(toBeFedBack,terminationCriterion);
+            case Closure(lambda(`v,`b),`s,`n):
+                final int max_num = ((MR_int)evalE(n,null)).get();
+                IterativeDataSet<FData> startOfIteration = eval(s,env).iterate(max_num);
+                DataSet<FData> toBeFedBack = eval(b,new Environment(v.toString(),new MR_flink(startOfIteration),env));
+                return startOfIteration.closeWith(toBeFedBack);
+            case let(`v,`u,`body):
+                return eval(body,new Environment(v.toString(),evalE(u,null),env));
+            case Let(`v,`u,`body):
+                return eval(body,new Environment(v.toString(),new MR_flink(eval(u,env)),env));
+            case If(`c,`x,`y):
+                if (((MR_bool)evalE(c,env)).get())
+                    return eval(x,env);
+                else return eval(y,env);
+            case `v:
+                if (!v.is_variable())
+                    fail;
+                MRData x = variable_lookup(v.toString(),env);
+                if (x != null)
+                    if (x instanceof MR_flink)
+                        return ((MR_flink)x).flink();
+                    else new Error("Variable "+v+" is of type: "+x);
+                x = variable_lookup(v.toString(),global_env);
+                if (x != null)
+                    if (x instanceof MR_flink)
+                        return ((MR_flink)x).flink();
+                    else new Error("Variable "+v+" is of type: "+x);
+                throw new Error("Variable "+v+" is not bound");
+            };
+            throw new Error("Cannot evaluate the Flink plan: "+e);
+        } catch (Error msg) {
+            if (!Config.trace)
+                throw new Error(msg.getMessage());
+            System.err.println(msg.getMessage());
+            msg.printStackTrace();
+            throw new Error("Evaluation error in: "+print_query(e));
+        } catch (Exception ex) {
+            System.err.println(ex.getMessage());
+            ex.printStackTrace();
+            throw new Error("Evaluation error in: "+print_query(e));
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkGeneratorInputFormat.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkGeneratorInputFormat.java b/flink/src/main/java/org/apache/mrql/FlinkGeneratorInputFormat.java
new file mode 100644
index 0000000..68969da
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkGeneratorInputFormat.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.flink.api.common.io.FileInputFormat;
+
+
+/** the FileInputFormat for data generators: it creates HDFS files, where each file contains
+ *  an (offset,size) pair that generates the range of values [offset,offset+size] */
+final public class FlinkGeneratorInputFormat extends FlinkMRQLFileInputFormat {
+    /** the Flink input format for this input */
+    public FileInputFormat<FData> inputFormat ( String path ) { return null; }
+}