You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrql.apache.org by fe...@apache.org on 2014/08/27 21:54:52 UTC
[1/2] MRQL-45: Add support for Apache Flink
Repository: incubator-mrql
Updated Branches:
refs/heads/master 5258de79a -> ea690387e
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkJsonParser.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkJsonParser.java b/flink/src/main/java/org/apache/mrql/FlinkJsonParser.java
new file mode 100644
index 0000000..114be7b
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkJsonParser.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** The JSON parser */
+public class FlinkJsonParser extends JsonFormatParser implements FlinkParser {
+ String[] tags; // split tags
+ FlinkJsonSplitter splitter;
+
+ public void open ( String file ) {
+ try {
+ splitter = new FlinkJsonSplitter(tags,file);
+ } catch (Exception e) {
+ throw new Error(e);
+ }
+ }
+
+ public void open ( FSDataInputStream fsin, long start, long end ) {
+ try {
+ splitter = new FlinkJsonSplitter(tags,fsin,start,end);
+ } catch (Exception e) {
+ throw new Error(e);
+ }
+ }
+
+ public String slice () {
+ if (splitter.hasNext()) {
+ DataOutputBuffer b = splitter.next();
+ return new String(b.getData(),0,b.getLength());
+ } else return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkJsonSplitter.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkJsonSplitter.java b/flink/src/main/java/org/apache/mrql/FlinkJsonSplitter.java
new file mode 100644
index 0000000..faaeff3
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkJsonSplitter.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.mrql.gen.*;
+import java_cup.runtime.Symbol;
+import java.util.Iterator;
+import java.io.*;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** Extract the JSON objects tagged by tags from a data split of the input stream (fsin) */
+final public class FlinkJsonSplitter implements Iterator<DataOutputBuffer> {
+ boolean in_memory;
+ FSDataInputStream fsin; // for HDFS processing
+ InputStream in; // for in-memory processing
+ JSONLex scanner;
+ String[] tags;
+ long start;
+ long end;
+ final DataOutputBuffer buffer;
+
+ FlinkJsonSplitter ( String[] tags, FSDataInputStream fsin, long start, long end ) {
+ in_memory = false;
+ this.tags = tags;
+ this.fsin = fsin;
+ this.end = end;
+ this.buffer = new DataOutputBuffer();
+ try {
+ fsin.seek(start);
+ this.start = (start == 0) ? start : sync(start);
+ fsin.seek(this.start);
+ scanner = new JSONLex(fsin);
+ } catch ( IOException e ) {
+ System.err.println("*** Cannot parse the data split: "+fsin);
+ }
+ }
+
+ FlinkJsonSplitter ( String[] tags, String file ) {
+ in_memory = true;
+ try {
+ in = new FileInputStream(file);
+ } catch ( Exception e ) {
+ throw new Error("Cannot open the file: "+file);
+ };
+ this.tags = tags;
+ this.buffer = new DataOutputBuffer();
+ scanner = new JSONLex(in);
+ }
+
+ private long sync ( long start ) {
+ try {
+ long first_quote = -1;
+ for ( long offset = 0; ; offset++ ) {
+ char c = (char)fsin.read();
+ if (c == '\"') {
+ if (first_quote >= 0)
+ if ((char)fsin.read() == ':')
+ return start+first_quote;
+ first_quote = offset;
+ }
+ }
+ } catch (IOException ex) {
+ return (long)0;
+ }
+ }
+
+ public boolean hasNext () {
+ try {
+ if (in_memory || start+scanner.char_pos() < end)
+ if (skip())
+ return store();
+ return false;
+ } catch (Exception e) {
+ System.err.println(e);
+ return false;
+ }
+ }
+
+ public DataOutputBuffer next () {
+ return buffer;
+ }
+
+ public void remove () { }
+
+ boolean is_start_tag ( String tagname ) {
+ if (tags == null)
+ return true;
+ for (String tag: tags)
+ if (tag.contentEquals(tagname))
+ return true;
+ return false;
+ }
+
+ /** skip until the beginning of a split element */
+ boolean skip () throws IOException {
+ while (true) {
+ Symbol s = scanner.next_token();
+ if (s.sym == jsym.EOF || (!in_memory && start+scanner.char_pos() >= end))
+ return false;
+ if (s.sym == jsym.STRING && is_start_tag((String)s.value)) {
+ String tag = (String)s.value;
+ if (scanner.next_token().sym == jsym.COLON) {
+ buffer.reset();
+ buffer.write('{');
+ buffer.write('\"');
+ for ( int i = 0; i < tag.length(); i++ )
+ buffer.write(tag.charAt(i));
+ buffer.write('\"');
+ buffer.write(':');
+ return true;
+ }
+ }
+ }
+ }
+
+ /** store one split element into the buffer; may cross split boundaries */
+ boolean store () throws IOException {
+ int nest = 0;
+ while (true) {
+ Symbol s = scanner.next_token();
+ if (s.sym == jsym.EOF)
+ return false;
+ if (s.sym == jsym.O_BEGIN || s.sym == jsym.A_BEGIN)
+ nest++;
+ else if (s.sym == jsym.O_END || s.sym == jsym.A_END)
+ nest--;
+ String text = scanner.text();
+ for ( int i = 0; i < text.length(); i++ )
+ buffer.write(text.charAt(i));
+ if (nest == 0) {
+ buffer.write('}');
+ return true;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkLineParser.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkLineParser.java b/flink/src/main/java/org/apache/mrql/FlinkLineParser.java
new file mode 100644
index 0000000..574286a
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkLineParser.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.*;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** A parser for line-oriented, character delimited text files (such as CVS) */
+final public class FlinkLineParser extends LineParser implements FlinkParser {
+ LineReader in;
+ byte[] line;
+
+ @Override
+ public void open ( FSDataInputStream fsin, long fstart, long fend ) {
+ in_memory = false;
+ start = fstart;
+ end = fend;
+ try {
+ in = new LineReader(fsin,start,end-start+1,maxLineLength);
+ if (false && start != 0) { // for all but the first data split, skip the first record
+ line = in.readLine();
+ if (line != null)
+ start += line.length;
+ };
+ pos = start;
+ } catch ( IOException e ) {
+ System.err.println("*** Cannot parse the data split: "+fsin);
+ start = end;
+ }
+ }
+
+ @Override
+ public String slice () {
+ try {
+ if (in_memory)
+ return buffered_in.readLine();
+ while (pos < end) {
+ line = in.readLine();
+ if (line == null || line.length == 0)
+ return null;
+ pos += line.length;
+ if (line.length < maxLineLength)
+ return new String(line);
+ };
+ return null;
+ } catch ( Exception e ) {
+ System.err.println("*** Cannot slice the text: "+e);
+ return "";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkMRQLFileInputFormat.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkMRQLFileInputFormat.java b/flink/src/main/java/org/apache/mrql/FlinkMRQLFileInputFormat.java
new file mode 100644
index 0000000..9877ceb
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkMRQLFileInputFormat.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.mrql.gen.*;
+import java.io.*;
+import org.apache.hadoop.fs.Path;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.core.fs.FileInputSplit;
+
+
+/** A superclass for all MRQL FileInputFormats */
+abstract public class FlinkMRQLFileInputFormat implements MRQLFileInputFormat {
+ public FlinkMRQLFileInputFormat () {}
+
+ private static DataSource read_directory ( String buffer ) {
+ try {
+ String[] s = buffer.split(DataSource.separator);
+ int n = Integer.parseInt(s[1]);
+ if (s[0].equals("Binary"))
+ return new BinaryDataSource(s[2],Plan.conf);
+ else if (s[0].equals("Generator"))
+ return new GeneratorDataSource(s[2],Plan.conf);
+ else if (s[0].equals("Text"))
+ return new FlinkParsedDataSource(s[3],DataSource.parserDirectory.get(s[2]),
+ ((Node)Tree.parse(s[4])).children());
+ else throw new Error("Unrecognized data source: "+s[0]);
+ } catch (Exception e) {
+ throw new Error(e);
+ }
+ }
+
+ /** encode the data source directory (to be serialized and distributed to clients) */
+ public static String encode_data_sources () {
+ return DataSource.dataSourceDirectory.toString();
+ }
+
+ /** restore the data source directory */
+ public static void restore_data_sources ( String data_sources ) {
+ if (Config.local_mode)
+ return;
+ DataSource.parserDirectory = new DataSource.ParserDirectory();
+ DataSource.parserDirectory.put("xml",FlinkXMLParser.class);
+ DataSource.parserDirectory.put("json",FlinkJsonParser.class);
+ DataSource.parserDirectory.put("line",FlinkLineParser.class);
+ DataSource.dataSourceDirectory = new DataSource.DataSourceDirectory();
+ for ( String s: data_sources.split("@@@") ) {
+ String[] p = s.split("===");
+ DataSource.dataSourceDirectory.put(p[0],read_directory(p[1]));
+ }
+ }
+
+ /** the Flink input format for this input */
+ abstract public FileInputFormat<FData> inputFormat ( String path );
+
+ /** materialize the input file into a memory Bag
+ * @param path the path directory
+ * @return a Bag that contains all data
+ */
+ public Bag materialize ( final Path path ) throws IOException {
+ final FileInputFormat<FData> sf = inputFormat(path.toString());
+ final FileInputSplit[] splits = sf.createInputSplits(1);
+ if (splits.length == 0)
+ return new Bag();
+ sf.open(splits[0]);
+ return new Bag(new BagIterator () {
+ int i = 0;
+ FData d = new FData();
+ public boolean hasNext () {
+ try {
+ while (sf.reachedEnd() && ++i < splits.length) {
+ sf.close();
+ sf.open(splits[i]);
+ };
+ return i < splits.length;
+ } catch (IOException ex) {
+ throw new Error("Cannot materialize a flink data source into a bag: "+ex);
+ }
+ }
+ public MRData next () {
+ try {
+ return sf.nextRecord(d).data();
+ } catch (IOException ex) {
+ throw new Error("Cannot materialize a flink data source into a bag: "+ex);
+ }
+ }
+ });
+ }
+
+ /** materialize the entire dataset into a Bag
+ * @param x the DataSet in HDFS to collect values from
+ * @param strip is not used in MapReduce mode
+ * @return the Bag that contains the collected values
+ */
+ public final Bag collect ( final DataSet x, boolean strip ) throws Exception {
+ Bag res = new Bag();
+ for ( DataSource s: x.source )
+ if (s.to_be_merged)
+ res = res.union(((FlinkDataSource)s).merge());
+ else res = res.union(s.inputFormat.newInstance().materialize(new Path(s.path)));
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkOutputFormat.java b/flink/src/main/java/org/apache/mrql/FlinkOutputFormat.java
new file mode 100644
index 0000000..6a496c9
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkOutputFormat.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.IOException;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.api.common.io.BinaryOutputFormat;
+
+public class FlinkOutputFormat extends BinaryOutputFormat<FData> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected void serialize ( FData data, DataOutputView out ) throws IOException {
+ data.write(out);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkParsedDataSource.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkParsedDataSource.java b/flink/src/main/java/org/apache/mrql/FlinkParsedDataSource.java
new file mode 100644
index 0000000..949f77b
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkParsedDataSource.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.mrql.gen.Trees;
+
+
+/** A data source for a text HDFS file along with the parser to parse it */
+public class FlinkParsedDataSource extends ParsedDataSource {
+ FlinkParsedDataSource ( String path,
+ Class<? extends Parser> parser,
+ Trees args ) {
+ this.source_num = -1;
+ this.path = path;
+ this.inputFormat = FlinkParsedInputFormat.class;
+ to_be_merged = false;
+ this.parser = parser;
+ this.args = args;
+ try {
+ dataSourceDirectory.put(path,this);
+ } catch (Exception ex) {
+ throw new Error("Cannot open file: "+path);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkParsedInputFormat.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkParsedInputFormat.java b/flink/src/main/java/org/apache/mrql/FlinkParsedInputFormat.java
new file mode 100644
index 0000000..b17127f
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkParsedInputFormat.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.*;
+import java.util.Iterator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.fs.FileInputSplit;
+
+
+/** A FileInputFormat for text files (CVS, XML, JSON, ...) */
+final public class FlinkParsedInputFormat extends FlinkMRQLFileInputFormat {
+ public FlinkParsedInputFormat () {}
+
+ public static final class ParsedInputFormat extends FileInputFormat<FData> {
+ String pathname;
+ transient FlinkParser parser;
+ transient Iterator<MRData> iter;
+ boolean eof;
+ String data_sources; // the encoded data source directory (to be serialized and distributed to clients)
+
+ public ParsedInputFormat ( String pathname ) {
+ this.pathname = pathname;
+ data_sources = encode_data_sources();
+ }
+
+ @Override
+ public void open ( FileInputSplit split ) throws IOException {
+ super.open(split);
+ restore_data_sources(data_sources);
+ ParsedDataSource ds = (ParsedDataSource)DataSource.get(split.getPath().toString(),Plan.conf);
+ try {
+ parser = (FlinkParser)ds.parser.newInstance();
+ } catch (Exception ex) {
+ throw new Error("Unrecognized parser: "+ds.parser);
+ };
+ parser.initialize(ds.args);
+ parser.open(stream,splitStart,splitStart+splitLength-1);
+ eof = false;
+ }
+
+ @Override
+ public FData nextRecord ( FData data ) throws IOException {
+ while (!eof && (iter == null || !iter.hasNext())) {
+ String line = parser.slice();
+ if (line == null) {
+ eof = true;
+ return null;
+ };
+ iter = parser.parse(line).iterator();
+ };
+ return new FData(iter.next());
+ }
+
+ @Override
+ public boolean reachedEnd () {
+ return eof;
+ }
+ }
+
+ /** the Flink input format for this input */
+ public FileInputFormat<FData> inputFormat ( String path ) {
+ return new ParsedInputFormat(path);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkParser.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkParser.java b/flink/src/main/java/org/apache/mrql/FlinkParser.java
new file mode 100644
index 0000000..9ac420f
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkParser.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** for a new text data source, you must implement a new parser */
+interface FlinkParser extends Parser {
+ public void open ( FSDataInputStream fsin, long start, long end );
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkXMLParser.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkXMLParser.java b/flink/src/main/java/org/apache/mrql/FlinkXMLParser.java
new file mode 100644
index 0000000..d8c61f9
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkXMLParser.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** A parser for line-oriented, character delimited text files (such as CVS) */
+final public class FlinkXMLParser extends XMLParser implements FlinkParser {
+ FlinkXMLSplitter splitter;
+
+ public void open ( String file ) {
+ try {
+ splitter = new FlinkXMLSplitter(tags,file);
+ } catch (Exception e) {
+ throw new Error(e);
+ }
+ }
+
+ public void open ( FSDataInputStream fsin, long start, long end ) {
+ try {
+ splitter = new FlinkXMLSplitter(tags,fsin,start,end);
+ } catch (Exception e) {
+ throw new Error(e);
+ }
+ }
+
+ public String slice () {
+ if (splitter.hasNext()) {
+ DataOutputBuffer b = splitter.next();
+ return new String(b.getData(),0,b.getLength());
+ } else return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkXMLSplitter.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkXMLSplitter.java b/flink/src/main/java/org/apache/mrql/FlinkXMLSplitter.java
new file mode 100644
index 0000000..c68211c
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkXMLSplitter.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.mrql.gen.*;
+import java.util.Iterator;
+import javax.xml.parsers.SAXParserFactory;
+import org.xml.sax.*;
+import java.io.*;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.flink.core.fs.FSDataInputStream;
+
+
+/** Extract the XML elements tagged by tags from a data split of the input stream (fsin)
+ * and store them in a buffer (to be parsed by SAX).
+ */
+final public class FlinkXMLSplitter implements Iterator<DataOutputBuffer> {
+ boolean in_memory;
+ FSDataInputStream fsin; // for HDFS processing
+ BufferedReader in; // for in-memory processing
+ String[] tags;
+ long start;
+ long end;
+ long pos;
+ StringBuffer tagname = new StringBuffer(100);
+ String start_tagname;
+ final DataOutputBuffer buffer;
+
+ FlinkXMLSplitter ( String[] tags, FSDataInputStream fsin, long start, long end ) {
+ in_memory = false;
+ this.tags = tags;
+ this.fsin = fsin;
+ this.start = start;
+ this.end = end;
+ pos = start;
+ this.buffer = new DataOutputBuffer();
+ try {
+ fsin.seek(start);
+ } catch ( IOException e ) {
+ System.err.println("*** Cannot parse the data split: "+fsin);
+ }
+ }
+
+ FlinkXMLSplitter ( String[] tags, String file ) {
+ in_memory = true;
+ try {
+ in = new BufferedReader(new InputStreamReader(new FileInputStream(file)),
+ 100000);
+ } catch ( Exception e ) {
+ throw new Error("Cannot open the file: "+file);
+ };
+ this.tags = tags;
+ this.buffer = new DataOutputBuffer();
+ }
+
+ public boolean hasNext () {
+ try {
+ if (in_memory || pos < end)
+ if (skip())
+ return store();
+ return false;
+ } catch (Exception e) {
+ System.err.println(e);
+ return false;
+ }
+ }
+
+ public DataOutputBuffer next () {
+ return buffer;
+ }
+
+ public void remove () { }
+
+ boolean is_start_tag () {
+ if (tags == null)
+ return true;
+ for (String tag: tags)
+ if (tag.contentEquals(tagname))
+ return true;
+ return false;
+ }
+
+ char read_tag () throws IOException {
+ tagname.setLength(0);
+ while (true) {
+ int b = in_memory ? in.read() : fsin.read();
+ if (b == -1)
+ return ' ';
+ else if (!Character.isLetterOrDigit(b) && b != ':' && b != '_')
+ return (char)b;
+ tagname.append((char)b);
+ }
+ }
+
+ /** skip until the beginning of a split element */
+ boolean skip () throws IOException {
+ while (true) {
+ int b = in_memory ? in.read() : fsin.read();
+ pos++;
+ if (b == -1 || (!in_memory && pos >= end))
+ return false;
+ else if (b == '<') {
+ b = read_tag();
+ pos += tagname.length()+1;
+ if (is_start_tag()) {
+ buffer.reset();
+ buffer.write('<');
+ for ( int i = 0; i < tagname.length(); i++ )
+ buffer.write(tagname.charAt(i));
+ buffer.write(b);
+ start_tagname = new String(tagname);
+ return true;
+ }
+ }
+ }
+ }
+
+ /** store one split element into the buffer; may cross split boundaries */
+ boolean store () throws IOException {
+ while (true) {
+ int b = in_memory ? in.read() : fsin.read();
+ pos++;
+ if (b == -1)
+ return false;
+ if (b == '&') { // don't validate external XML entities
+ buffer.write('&');buffer.write('a');buffer.write('m');buffer.write('p');buffer.write(';');
+ } else buffer.write(b);
+ if (b == '<') {
+ b = in_memory ? in.read() : fsin.read();
+ pos++;
+ buffer.write(b);
+ if (b == '/') {
+ b = read_tag();
+ pos += tagname.length()+1;
+ for ( int i = 0; i < tagname.length(); i++ )
+ buffer.write(tagname.charAt(i));
+ buffer.write(b);
+ if (start_tagname.contentEquals(tagname)) {
+ while (b != '>') {
+ b = fsin.read();
+ pos++;
+ buffer.write(b);
+ };
+ return true;
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/LineReader.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/LineReader.java b/flink/src/main/java/org/apache/mrql/LineReader.java
new file mode 100644
index 0000000..28bb0d1
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/LineReader.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * This file is based on source code from the Hadoop Project (http://hadoop.apache.org/), licensed by the Apache
+ * Software Foundation (ASF) under the Apache License, Version 2.0.
+ */
+
+package org.apache.mrql;
+
+import java.io.IOException;
+
+import org.apache.flink.core.fs.FSDataInputStream;
+
+public class LineReader {
+
+ private static final int CR = '\r';
+
+ private static final int LF = '\n';
+
+ private FSDataInputStream stream;
+
+ private byte[] readBuffer;
+
+ private byte[] wrapBuffer;
+
+ private long lengthLeft;
+
+ private int readPos;
+
+ private int limit;
+
+ private boolean overLimit;
+
+ public LineReader(final FSDataInputStream strm, final long start, final long length, final int buffersize)
+ throws IOException {
+ this.stream = strm;
+ this.readBuffer = new byte[buffersize];
+ this.wrapBuffer = new byte[256];
+
+ this.lengthLeft = length;
+ this.readPos = 0;
+ this.overLimit = false;
+
+ if (start != 0) {
+ strm.seek(start);
+ readLine();
+ } else {
+ fillBuffer();
+ }
+ }
+
+ private final boolean fillBuffer() throws IOException {
+
+ int toRead = this.lengthLeft > this.readBuffer.length ? this.readBuffer.length : (int) this.lengthLeft;
+ if (this.lengthLeft <= 0) {
+ toRead = this.readBuffer.length;
+ this.overLimit = true;
+ }
+
+ int read = this.stream.read(this.readBuffer, 0, toRead);
+
+ if (read == -1) {
+ this.stream.close();
+ this.stream = null;
+ return false;
+ } else {
+ this.lengthLeft -= read;
+ this.readPos = 0;
+ this.limit = read;
+ return true;
+ }
+
+ }
+
+ public void close() throws IOException {
+ this.wrapBuffer = null;
+ this.readBuffer = null;
+ if (this.stream != null) {
+ this.stream.close();
+ }
+ }
+
+ public byte[] readLine() throws IOException {
+ if (this.stream == null || this.overLimit) {
+ return null;
+ }
+
+ int curr = 0;
+ int countInWrapBuffer = 0;
+
+ while (true) {
+ if (this.readPos >= this.limit) {
+ if (!fillBuffer()) {
+ if (countInWrapBuffer > 0) {
+ byte[] tmp = new byte[countInWrapBuffer];
+ System.arraycopy(this.wrapBuffer, 0, tmp, 0, countInWrapBuffer);
+ return tmp;
+ } else {
+ return null;
+ }
+ }
+ }
+
+ int startPos = this.readPos;
+ int count = 0;
+
+ while (this.readPos < this.limit && (curr = this.readBuffer[this.readPos++]) != LF) {
+ }
+
+ // check why we dropped out
+ if (curr == LF) {
+ // line end
+ count = this.readPos - startPos - 1;
+ if (this.readPos == 1 && countInWrapBuffer > 0 && this.wrapBuffer[countInWrapBuffer - 1] == CR) {
+ countInWrapBuffer--;
+ } else if (this.readPos > startPos + 1 && this.readBuffer[this.readPos - 2] == CR) {
+ count--;
+ }
+
+ // copy to byte array
+ if (countInWrapBuffer > 0) {
+ byte[] end = new byte[countInWrapBuffer + count];
+ System.arraycopy(this.wrapBuffer, 0, end, 0, countInWrapBuffer);
+ System.arraycopy(this.readBuffer, 0, end, countInWrapBuffer, count);
+ return end;
+ } else {
+ byte[] end = new byte[count];
+ System.arraycopy(this.readBuffer, startPos, end, 0, count);
+ return end;
+ }
+ } else {
+ count = this.limit - startPos;
+
+ // buffer exhausted
+ while (this.wrapBuffer.length - countInWrapBuffer < count) {
+ // reallocate
+ byte[] tmp = new byte[this.wrapBuffer.length * 2];
+ System.arraycopy(this.wrapBuffer, 0, tmp, 0, countInWrapBuffer);
+ this.wrapBuffer = tmp;
+ }
+
+ System.arraycopy(this.readBuffer, startPos, this.wrapBuffer, countInWrapBuffer, count);
+ countInWrapBuffer += count;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/MR_flink.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/MR_flink.java b/flink/src/main/java/org/apache/mrql/MR_flink.java
new file mode 100644
index 0000000..511cfb9
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/MR_flink.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
+import org.apache.flink.api.java.DataSet;
+
+
+/** a wrapper of a DataSet<FData> as an MRData */
+final public class MR_flink extends MRData {
+ public DataSet<FData> flink;
+
+ public MR_flink ( DataSet<FData> d ) { flink = d; }
+
+ public void materializeAll () {};
+
+ public DataSet<FData> flink () { return flink; }
+
+ final public void write ( DataOutput out ) throws IOException {
+ throw new Error("DataSets are not serializable");
+ }
+
+ public void readFields ( DataInput in ) throws IOException {
+ throw new Error("DataSets are not serializable");
+ }
+
+ public int compareTo ( MRData x ) {
+ throw new Error("DataSets cannot be compared");
+ }
+
+ public boolean equals ( Object x ) {
+ throw new Error("DataSets cannot be compared");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/test/java/org/apache/mrql/FlinkEvaluatorLocalModeTest.java
----------------------------------------------------------------------
diff --git a/flink/src/test/java/org/apache/mrql/FlinkEvaluatorLocalModeTest.java b/flink/src/test/java/org/apache/mrql/FlinkEvaluatorLocalModeTest.java
new file mode 100644
index 0000000..9e3b030
--- /dev/null
+++ b/flink/src/test/java/org/apache/mrql/FlinkEvaluatorLocalModeTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+public class FlinkEvaluatorLocalModeTest extends EvaluatorTest {
+
+ public void tearDown() throws IOException {
+ super.tearDown();
+ Plan.clean();
+ }
+
+ @Override
+ protected Evaluator createEvaluator() throws Exception {
+ Configuration conf = null;
+
+ Config.bsp_mode = false;
+ Config.spark_mode = false;
+ Config.flink_mode = true;
+ Config.map_reduce_mode = false;
+
+ Evaluator.evaluator = new FlinkEvaluator();
+
+ Config.quiet_execution = true;
+
+ String[] args = new String[] { "-local", "-flink" };
+
+ conf = Evaluator.evaluator.new_configuration();
+ GenericOptionsParser gop = new GenericOptionsParser(conf, args);
+ conf = gop.getConfiguration();
+
+ args = gop.getRemainingArgs();
+
+ Config.hadoop_mode = true;
+ Config.testing = true;
+ Config.parse_args(args, conf);
+
+ Evaluator.evaluator.init(conf);
+
+ return Evaluator.evaluator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/test/java/org/apache/mrql/FlinkQueryLocalModeTest.java
----------------------------------------------------------------------
diff --git a/flink/src/test/java/org/apache/mrql/FlinkQueryLocalModeTest.java b/flink/src/test/java/org/apache/mrql/FlinkQueryLocalModeTest.java
new file mode 100644
index 0000000..81a122b
--- /dev/null
+++ b/flink/src/test/java/org/apache/mrql/FlinkQueryLocalModeTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+public class FlinkQueryLocalModeTest extends QueryTest {
+
+ public void tearDown() throws IOException {
+ super.tearDown();
+ Plan.clean();
+ }
+
+ @Override
+ protected Evaluator createEvaluator() throws Exception {
+ Configuration conf = new Configuration();
+
+ Config.bsp_mode = false;
+ Config.spark_mode = false;
+ Config.flink_mode = true;
+ Config.map_reduce_mode = false;
+
+ Evaluator.evaluator = new FlinkEvaluator();
+
+ Config.quiet_execution = true;
+
+ String[] args = new String[] { "-local", "-flink" };
+
+ conf = Evaluator.evaluator.new_configuration();
+ GenericOptionsParser gop = new GenericOptionsParser(conf, args);
+ conf = gop.getConfiguration();
+
+ args = gop.getRemainingArgs();
+
+ Config.hadoop_mode = false;
+ Config.testing = true;
+ Config.parse_args(args, conf);
+
+ Evaluator.evaluator.init(conf);
+
+ return Evaluator.evaluator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 70fb038..bf5e2df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,7 +23,7 @@
<version>0.9.4-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache MRQL parent POM</name>
- <description>Apache MRQL is a query processing and optimization system for large-scale, distributed data analysis, built on top of Apache Hadoop, Hama, and Spark</description>
+ <description>Apache MRQL is a query processing and optimization system for large-scale, distributed data analysis, built on top of Apache Hadoop, Hama, Spark, and Flink</description>
<url>http://mrql.incubator.apache.org/</url>
<inceptionYear>2013</inceptionYear>
<licenses>
@@ -47,8 +47,9 @@
<hadoop.version>1.2.1</hadoop.version>
<yarn.version>2.2.0</yarn.version>
<hama.version>0.6.3</hama.version>
- <spark.version>1.0.0</spark.version>
+ <spark.version>1.0.2</spark.version>
<scala.version>2.10</scala.version>
+ <flink.version>0.6-incubating</flink.version>
<skipTests>true</skipTests>
</properties>
@@ -58,6 +59,7 @@
<module>mapreduce</module>
<module>bsp</module>
<module>spark</module>
+ <module>flink</module>
<module>dist</module>
</modules>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen b/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
index 23c29f0..b7f43b4 100644
--- a/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
+++ b/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
@@ -97,7 +97,7 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
final public void initialize_query () {
Plan.distribute_compiled_arguments(Plan.conf);
- if (Config.compile_functional_arguments)
+ if (spark_context != null && Config.compile_functional_arguments)
spark_context.addJar(Plan.conf.get("mrql.jar.path"));
}
@@ -169,21 +169,21 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
@Override
final public void dump ( String file, Tree type, MRData data ) throws Exception {
if (data instanceof MR_dataset && ((MR_dataset)data).dataset().source.get(0) instanceof RDDDataSource) {
- Path path = new Path(file);
- FileSystem fs = path.getFileSystem(Plan.conf);
- PrintStream ftp = new PrintStream(fs.create(path.suffix(".type")));
+ Path path = new Path(file);
+ FileSystem fs = path.getFileSystem(Plan.conf);
+ PrintStream ftp = new PrintStream(fs.create(path.suffix(".type")));
fs.delete(path,true);
- ftp.print("2@"+type.toString()+"\n");
- ftp.close();
+ ftp.print("2@"+type.toString()+"\n");
+ ftp.close();
// in Spark mode, a dataset has always one RDD datasource
- JavaRDD<MRData> rdd = ((RDDDataSource)((MR_dataset)data).dataset().source.get(0)).rdd.cache();
+ JavaRDD<MRData> rdd = ((RDDDataSource)((MR_dataset)data).dataset().source.get(0)).rdd.cache();
rdd.mapToPair(new PairFunction<MRData,MRContainer,MRContainer>() {
final MRContainer zero = new MRContainer(new MR_int(0));
public Tuple2<MRContainer,MRContainer> call ( MRData value ) {
return new Tuple2<MRContainer,MRContainer>(zero,new MRContainer(value));
}
}).saveAsHadoopFile(file,MRContainer.class,MRContainer.class,SequenceFileOutputFormat.class);
- } else super.dump(file,type,data);
+ } else super.dump(file,type,data);
}
/** dump MRQL data into text a CVS file */
@@ -191,26 +191,26 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
final public void dump_text ( String file, Tree type, MRData data ) throws Exception {
if (data instanceof MR_dataset && ((MR_dataset)data).dataset().source.get(0) instanceof RDDDataSource) {
// in Spark mode, a dataset has always one RDD datasource
- JavaRDD<MRData> rdd = ((RDDDataSource)((MR_dataset)data).dataset().source.get(0)).rdd.cache();
- int ps = Config.max_bag_size_print;
- Config.max_bag_size_print = -1;
- match type {
- case `T(tuple(...tps)):
- final Trees ts = tps;
- rdd.map(new Function<MRData,String>() {
+ JavaRDD<MRData> rdd = ((RDDDataSource)((MR_dataset)data).dataset().source.get(0)).rdd.cache();
+ int ps = Config.max_bag_size_print;
+ Config.max_bag_size_print = -1;
+ match type {
+ case `T(tuple(...tps)):
+ final Trees ts = tps;
+ rdd.map(new Function<MRData,String>() {
public String call ( MRData value ) {
- Tuple t = (Tuple)value;
- String s = Printer.print(t.get((short)0),ts.nth(0));
- for ( short i = 1; i < t.size(); i++ )
- s += ","+Printer.print(t.get(i),ts.nth(i));
- return s;
- }
- }).saveAsTextFile(file);
- case _:
- rdd.saveAsTextFile(file);
- };
- Config.max_bag_size_print = ps;
- } else super.dump_text(file,type,data);
+ Tuple t = (Tuple)value;
+ String s = Printer.print(t.get((short)0),ts.nth(0));
+ for ( short i = 1; i < t.size(); i++ )
+ s += ","+Printer.print(t.get(i),ts.nth(i));
+ return s;
+ }
+ }).saveAsTextFile(file);
+ case _:
+ rdd.saveAsTextFile(file);
+ };
+ Config.max_bag_size_print = ps;
+ } else super.dump_text(file,type,data);
}
private static Function2<MRData,MRData,MRData> accumulator ( final Tree acc_fnc, final Environment env ) {
@@ -554,7 +554,7 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
}
});
case GroupByJoin(`kx,`ky,`gx,`gy,`acc,`zero,`r,`x,`y,`o):
- final int n = (int)Math.floor(Math.sqrt(Config.nodes));
+ final int n = Math.max(2,(int)Math.floor(Math.sqrt(Config.nodes)));
final int m = n;
// System.err.println("Using a groupBy join on a "+n+"*"+m+" grid of partitions");
final org.apache.mrql.Function fkx = evalF(kx,env);
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/spark/src/test/java/org/apache/mrql/SparkEvaluatorInMemoryTest.java
----------------------------------------------------------------------
diff --git a/spark/src/test/java/org/apache/mrql/SparkEvaluatorInMemoryTest.java b/spark/src/test/java/org/apache/mrql/SparkEvaluatorInMemoryTest.java
deleted file mode 100644
index 39c4ad4..0000000
--- a/spark/src/test/java/org/apache/mrql/SparkEvaluatorInMemoryTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.mrql;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class SparkEvaluatorInMemoryTest extends EvaluatorTest {
-
- @Override
- protected Evaluator createEvaluator() throws Exception {
- Configuration conf = null;
-
- Config.bsp_mode = false;
- Config.spark_mode = true;
- Config.map_reduce_mode = false;
-
- Evaluator.evaluator = new SparkEvaluator();
-
- Config.quiet_execution = true;
- Config.hadoop_mode = false;
- Config.testing = true;
- Config.parse_args(new String[] {"-spark"}, conf);
-
- Evaluator.evaluator.init(conf);
-
- return Evaluator.evaluator;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/spark/src/test/java/org/apache/mrql/SparkQueryInMemoryTest.java
----------------------------------------------------------------------
diff --git a/spark/src/test/java/org/apache/mrql/SparkQueryInMemoryTest.java b/spark/src/test/java/org/apache/mrql/SparkQueryInMemoryTest.java
deleted file mode 100644
index b35365d..0000000
--- a/spark/src/test/java/org/apache/mrql/SparkQueryInMemoryTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.mrql;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class SparkQueryInMemoryTest extends QueryTest {
-
- @Override
- protected Evaluator createEvaluator() throws Exception {
- Configuration conf = null;
-
- Config.bsp_mode = false;
- Config.spark_mode = true;
- Config.map_reduce_mode = false;
-
- Evaluator.evaluator = new SparkEvaluator();
-
- Config.quiet_execution = true;
- Config.hadoop_mode = false;
- Config.testing = true;
- Config.parse_args(new String[] { "-spark" }, conf);
-
- Evaluator.evaluator.init(conf);
-
- return Evaluator.evaluator;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/spark/src/test/java/org/apache/mrql/SparkQueryLocalModeTest.java
----------------------------------------------------------------------
diff --git a/spark/src/test/java/org/apache/mrql/SparkQueryLocalModeTest.java b/spark/src/test/java/org/apache/mrql/SparkQueryLocalModeTest.java
index 72b32e4..7c2812d 100644
--- a/spark/src/test/java/org/apache/mrql/SparkQueryLocalModeTest.java
+++ b/spark/src/test/java/org/apache/mrql/SparkQueryLocalModeTest.java
@@ -54,7 +54,7 @@ public class SparkQueryLocalModeTest extends QueryTest {
Config.parse_args(args, conf);
Evaluator.evaluator.init(conf);
-
+
return Evaluator.evaluator;
}
}
[2/2] git commit: MRQL-45: Add support for Apache Flink
Posted by fe...@apache.org.
MRQL-45: Add support for Apache Flink
Project: http://git-wip-us.apache.org/repos/asf/incubator-mrql/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mrql/commit/ea690387
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mrql/tree/ea690387
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mrql/diff/ea690387
Branch: refs/heads/master
Commit: ea690387ea1db9f7924ffe0670bb9e167b16c83c
Parents: 5258de7
Author: fegaras <fe...@cse.uta.edu>
Authored: Wed Aug 27 14:53:49 2014 -0500
Committer: fegaras <fe...@cse.uta.edu>
Committed: Wed Aug 27 14:53:49 2014 -0500
----------------------------------------------------------------------
bin/mrql.flink | 58 ++
conf/mrql-env.sh | 28 +-
.../org/apache/mrql/AlgebraicOptimization.gen | 8 +-
core/src/main/java/org/apache/mrql/Config.java | 7 +
core/src/main/java/org/apache/mrql/Inv.java | 2 +
.../main/java/org/apache/mrql/LineParser.gen | 2 +-
.../main/java/org/apache/mrql/MRContainer.java | 2 +-
core/src/main/java/org/apache/mrql/MR_bool.java | 2 +
core/src/main/java/org/apache/mrql/MR_byte.java | 3 +
core/src/main/java/org/apache/mrql/MR_char.java | 2 +
.../main/java/org/apache/mrql/MR_double.java | 2 +
.../src/main/java/org/apache/mrql/MR_float.java | 2 +
core/src/main/java/org/apache/mrql/MR_int.java | 2 +
core/src/main/java/org/apache/mrql/MR_long.java | 2 +
.../src/main/java/org/apache/mrql/MR_short.java | 2 +
.../main/java/org/apache/mrql/MR_string.java | 2 +
core/src/main/java/org/apache/mrql/Main.java | 9 +-
.../java/org/apache/mrql/ParsedDataSource.java | 2 +
.../java/org/apache/mrql/PlanGeneration.gen | 8 +
.../java/org/apache/mrql/SystemFunctions.java | 21 +-
core/src/main/java/org/apache/mrql/Tuple.java | 2 +
core/src/main/java/org/apache/mrql/Union.java | 2 +
.../main/java/org/apache/mrql/XMLParser.java | 2 +-
.../test/java/org/apache/mrql/QueryTest.java | 2 +-
flink/pom.xml | 148 ++++
flink/src/main/java/org/apache/mrql/FData.java | 73 ++
.../java/org/apache/mrql/FDataContainer.java | 39 ++
.../org/apache/mrql/FlinkBinaryInputFormat.java | 44 ++
.../java/org/apache/mrql/FlinkDataSource.java | 290 ++++++++
.../java/org/apache/mrql/FlinkEvaluator.gen | 691 +++++++++++++++++++
.../apache/mrql/FlinkGeneratorInputFormat.java | 28 +
.../java/org/apache/mrql/FlinkJsonParser.java | 51 ++
.../java/org/apache/mrql/FlinkJsonSplitter.java | 153 ++++
.../java/org/apache/mrql/FlinkLineParser.java | 67 ++
.../apache/mrql/FlinkMRQLFileInputFormat.java | 118 ++++
.../java/org/apache/mrql/FlinkOutputFormat.java | 31 +
.../org/apache/mrql/FlinkParsedDataSource.java | 40 ++
.../org/apache/mrql/FlinkParsedInputFormat.java | 85 +++
.../main/java/org/apache/mrql/FlinkParser.java | 26 +
.../java/org/apache/mrql/FlinkXMLParser.java | 50 ++
.../java/org/apache/mrql/FlinkXMLSplitter.java | 165 +++++
.../main/java/org/apache/mrql/LineReader.java | 164 +++++
.../src/main/java/org/apache/mrql/MR_flink.java | 51 ++
.../mrql/FlinkEvaluatorLocalModeTest.java | 61 ++
.../apache/mrql/FlinkQueryLocalModeTest.java | 61 ++
pom.xml | 6 +-
.../java/org/apache/mrql/SparkEvaluator.gen | 56 +-
.../apache/mrql/SparkEvaluatorInMemoryTest.java | 43 --
.../org/apache/mrql/SparkQueryInMemoryTest.java | 43 --
.../apache/mrql/SparkQueryLocalModeTest.java | 2 +-
50 files changed, 2626 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/bin/mrql.flink
----------------------------------------------------------------------
diff --git a/bin/mrql.flink b/bin/mrql.flink
new file mode 100755
index 0000000..9a1c272
--- /dev/null
+++ b/bin/mrql.flink
@@ -0,0 +1,58 @@
+#!/bin/bash
+#--------------------------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#--------------------------------------------------------------------------------
+#
+# run Apache MRQL in Apache Flink mode
+#
+#--------------------------------------------------------------------------------
+
+MRQL_HOME="$(cd `dirname $0`/..; pwd -P)"
+
+. "$MRQL_HOME/conf/mrql-env.sh"
+
+GEN_JAR=`ls "$MRQL_HOME"/lib/mrql-gen-*.jar`
+CORE_JAR=`ls "$MRQL_HOME"/lib/mrql-core-*.jar`
+MRQL_JAR=`ls "$MRQL_HOME"/lib/mrql-flink-*.jar`
+FULL_JAR="$MRQL_HOME/lib/mrql-all-flink.jar"
+
+export FLINK_HOME FLINK_JAR FLINK_MASTER
+
+if (! [ -a $FULL_JAR ]); then
+ rm -rf "$MRQL_HOME/tmp/classes"
+ mkdir -p "$MRQL_HOME/tmp/classes"
+ pushd $MRQL_HOME/tmp/classes > /dev/null
+ $JAVA_HOME/bin/jar xf $CUP_JAR
+ $JAVA_HOME/bin/jar xf $JLINE_JAR
+ $JAVA_HOME/bin/jar xf $GEN_JAR
+ $JAVA_HOME/bin/jar xf $CORE_JAR
+ $JAVA_HOME/bin/jar xf $MRQL_JAR
+ cd ..
+ $JAVA_HOME/bin/jar cf $FULL_JAR -C classes/ .
+ popd > /dev/null
+fi
+
+if [ "$1" == "-dist" ]; then
+ args=" -flink $*"
+ args=${args// /\!}
+ $FLINK_HOME/bin/flink run $FULL_JAR -c org.apache.mrql.Main -v args $args
+else
+ $JAVA_HOME/bin/java -classpath "$FULL_JAR:$FLINK_JAR" org.apache.mrql.Main -flink $*
+fi
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/conf/mrql-env.sh
----------------------------------------------------------------------
diff --git a/conf/mrql-env.sh b/conf/mrql-env.sh
index 90fa261..1635334 100644
--- a/conf/mrql-env.sh
+++ b/conf/mrql-env.sh
@@ -46,7 +46,7 @@ JLINE_JAR=${HOME}/.m2/repository/jline/jline/1.0/jline-1.0.jar
# Required: Hadoop configuration. Supports versions 1.x and 2.x (YARN)
-HADOOP_VERSION=1.2.1
+HADOOP_VERSION=2.2.0
# The Hadoop installation directory
HADOOP_HOME=${HOME}/hadoop-${HADOOP_VERSION}
# The Hadoop job tracker (as defined in mapred-site.xml)
@@ -64,15 +64,17 @@ BSP_MASTER_ADDRESS=localhost:40000
HAMA_ZOOKEEPER_QUORUM=localhost
-# Optional: Spark configuration. Supports version 1.0.0 only
+# Optional: Spark configuration. Supports versions 1.0.0 and 1.0.2 only
# (Spark versions 0.8.1, 0.9.0, and 0.9.1 are supported by MRQL 0.9.0)
# Use the Spark prebuilts bin-hadoop1 or bin-hadoop2 (Yarn)
# Tested in local, standalone deploy, and Yarn modes
-SPARK_HOME=${HOME}/spark-1.0.0-bin-hadoop1
-# URI of the Spark master node (to run Spark on a YARN cluster, set it to "yarn-client")
-SPARK_MASTER=spark://crete:7077
+SPARK_HOME=${HOME}/spark-1.0.2-bin-hadoop2
+# URI of the Spark master node:
+# to run Spark on Standalone Mode, set it to spark://`hostname`:7077
+# to run Spark on a YARN cluster, set it to "yarn-client"
+SPARK_MASTER=yarn-client
# For a Yarn cluster set it to the number of workers to start on,
-# for local/standalone set it to 1
+# for local/standalone set it to 1
SPARK_WORKER_INSTANCES=1
# Number of cores for the workers
SPARK_WORKER_CORES=2
@@ -81,10 +83,24 @@ SPARK_WORKER_MEMORY=1G
# Memory for Master (e.g. 1000M, 2G)
SPARK_MASTER_MEMORY=512M
+
+# Optional: Flink configuration. Supports version 0.6-incubating
+# Note: for yarn, set yarn.nodemanager.vmem-check-enabled to false in yarn-site.xml
+FLINK_VERSION=yarn-0.6-incubating
+# Flink installation directory
+FLINK_HOME=${HOME}/flink-${FLINK_VERSION}
+# Flink JobManager (on a yarn cluster, it is printed when you run yarn-session.sh)
+if [ "$FLINK_MASTER" = "" ]; then
+ FLINK_MASTER=`hostname`
+fi
+
+
# Claspaths
HAMA_JAR=${HAMA_HOME}/hama-core-${HAMA_VERSION}.jar
+FLINK_JAR=`ls ${FLINK_HOME}/lib/flink-*.jar`
+
# YARN-enabled assembly jar
if [[ -d ${SPARK_HOME}/assembly/target ]]; then
# old Spark (0.x)
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/AlgebraicOptimization.gen
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/AlgebraicOptimization.gen b/core/src/main/java/org/apache/mrql/AlgebraicOptimization.gen
index 9126de6..b89dd3c 100644
--- a/core/src/main/java/org/apache/mrql/AlgebraicOptimization.gen
+++ b/core/src/main/java/org/apache/mrql/AlgebraicOptimization.gen
@@ -69,13 +69,17 @@ public class AlgebraicOptimization extends Simplification {
case mapReduce2(`mx,`my,`r,cmap(lambda(`v,`b),`X),`Y,`o):
match X {
case groupBy(...): ;
- case _: return translate(#<mapReduce2(lambda(`v,cmap(`mx,`b)),`my,`r,`X,`Y,`o)>);
+ case _:
+ Tree nmx = simplify(#<cmap(`mx,`b)>);
+ return translate(#<mapReduce2(lambda(`v,`nmx),`my,`r,`X,`Y,`o)>);
};
fail
case mapReduce2(`mx,`my,`r,`X,cmap(lambda(`v,`b),`Y),`o):
match Y {
case groupBy(...): ;
- case _: return translate(#<mapReduce2(`mx,lambda(`v,cmap(`my,`b)),`r,`X,`Y,`o)>);
+ case _:
+ Tree nmy = simplify(#<cmap(`my,`b)>);
+ return translate(#<mapReduce2(`mx,lambda(`v,`nmy),`r,`X,`Y,`o)>);
};
fail
case crossProduct(`mx,`my,`r,cmap(lambda(`vx,`bx),`X),cmap(lambda(`vy,`by),`Y)):
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/Config.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/Config.java b/core/src/main/java/org/apache/mrql/Config.java
index f46319a..4c4aff9 100644
--- a/core/src/main/java/org/apache/mrql/Config.java
+++ b/core/src/main/java/org/apache/mrql/Config.java
@@ -39,6 +39,8 @@ final public class Config {
public static boolean bsp_mode = false;
// true, for Spark mode
public static boolean spark_mode = false;
+ // true, for Flink mode
+ public static boolean flink_mode = false;
// if true, it process the input interactively
public static boolean interactive = true;
// compile the MR functional arguments to Java bytecode at run-time
@@ -91,6 +93,7 @@ final public class Config {
conf.setBoolean("mrql.map.reduce.mode",map_reduce_mode);
conf.setBoolean("mrql.bsp.mode",bsp_mode);
conf.setBoolean("mrql.spark.mode",spark_mode);
+ conf.setBoolean("mrql.flink.mode",flink_mode);
conf.setBoolean("mrql.interactive",interactive);
conf.setBoolean("mrql.compile.functional.arguments",compile_functional_arguments);
conf.setBoolean("mrql.trace",trace);
@@ -121,6 +124,7 @@ final public class Config {
map_reduce_mode = conf.getBoolean("mrql.map.reduce.mode",map_reduce_mode);
bsp_mode = conf.getBoolean("mrql.bsp.mode",bsp_mode);
spark_mode = conf.getBoolean("mrql.spark.mode",spark_mode);
+ flink_mode = conf.getBoolean("mrql.flink.mode",flink_mode);
interactive = conf.getBoolean("mrql.interactive",interactive);
compile_functional_arguments = conf.getBoolean("mrql.compile.functional.arguments",compile_functional_arguments);
trace = conf.getBoolean("mrql.trace",trace);
@@ -170,6 +174,9 @@ final public class Config {
} else if (args[i].equals("-spark")) {
spark_mode = true;
i++;
+ } else if (args[i].equals("-flink")) {
+ flink_mode = true;
+ i++;
} else if (args[i].equals("-bsp_tasks")) {
if (++i >= args.length && Integer.parseInt(args[i]) < 1)
throw new Error("Expected max number of bsp tasks > 1");
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/Inv.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/Inv.java b/core/src/main/java/org/apache/mrql/Inv.java
index 3faa7fe..99db099 100644
--- a/core/src/main/java/org/apache/mrql/Inv.java
+++ b/core/src/main/java/org/apache/mrql/Inv.java
@@ -25,6 +25,8 @@ import java.io.DataOutput;
final public class Inv extends MRData {
private MRData value;
+ public Inv () {}
+
Inv ( MRData v ) { value = v; }
public void materializeAll () { value.materializeAll(); };
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/LineParser.gen
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/LineParser.gen b/core/src/main/java/org/apache/mrql/LineParser.gen
index fb1e068..5619b3f 100644
--- a/core/src/main/java/org/apache/mrql/LineParser.gen
+++ b/core/src/main/java/org/apache/mrql/LineParser.gen
@@ -26,7 +26,7 @@ import org.apache.hadoop.util.LineReader;
/** A parser for line-oriented, character delimited text files (such as CVS) */
-final public class LineParser implements Parser {
+public class LineParser implements Parser {
final static int maxLineLength = 1000;
boolean in_memory;
FSDataInputStream fsin; // for HDFS processing
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MRContainer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MRContainer.java b/core/src/main/java/org/apache/mrql/MRContainer.java
index d72c452..42caa5c 100644
--- a/core/src/main/java/org/apache/mrql/MRContainer.java
+++ b/core/src/main/java/org/apache/mrql/MRContainer.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.io.WritableComparable;
/** A container for MRData that implements read (the deserializer) */
-final public class MRContainer implements WritableComparable<MRContainer>, Serializable {
+public class MRContainer implements WritableComparable<MRContainer>, Serializable {
transient MRData data;
public final static byte BOOLEAN = 0, BYTE = 1, SHORT = 2, INT = 3, LONG = 4,
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_bool.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_bool.java b/core/src/main/java/org/apache/mrql/MR_bool.java
index d40573f..ef3e453 100644
--- a/core/src/main/java/org/apache/mrql/MR_bool.java
+++ b/core/src/main/java/org/apache/mrql/MR_bool.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.fs.*;
final public class MR_bool extends MRData {
private boolean value;
+ public MR_bool () {}
+
public MR_bool ( boolean i ) { value = i; }
public void materializeAll () {};
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_byte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_byte.java b/core/src/main/java/org/apache/mrql/MR_byte.java
index e5c319e..489843d 100644
--- a/core/src/main/java/org/apache/mrql/MR_byte.java
+++ b/core/src/main/java/org/apache/mrql/MR_byte.java
@@ -28,7 +28,10 @@ import org.apache.hadoop.fs.*;
final public class MR_byte extends MRData {
private byte value;
+ public MR_byte () {}
+
public MR_byte ( byte i ) { value = i; }
+
public MR_byte ( int i ) { value = (byte)i; }
public void materializeAll () {};
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_char.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_char.java b/core/src/main/java/org/apache/mrql/MR_char.java
index f0e9721..7a70996 100644
--- a/core/src/main/java/org/apache/mrql/MR_char.java
+++ b/core/src/main/java/org/apache/mrql/MR_char.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
final public class MR_char extends MRData {
private char value;
+ public MR_char () {}
+
public MR_char ( char i ) { value = i; }
public void materializeAll () {};
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_double.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_double.java b/core/src/main/java/org/apache/mrql/MR_double.java
index 567917c..c0e7d57 100644
--- a/core/src/main/java/org/apache/mrql/MR_double.java
+++ b/core/src/main/java/org/apache/mrql/MR_double.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
final public class MR_double extends MRData {
private double value;
+ public MR_double () {}
+
public MR_double ( double x ) { value = x; }
public void materializeAll () {};
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_float.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_float.java b/core/src/main/java/org/apache/mrql/MR_float.java
index d8dda38..121a37c 100644
--- a/core/src/main/java/org/apache/mrql/MR_float.java
+++ b/core/src/main/java/org/apache/mrql/MR_float.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
final public class MR_float extends MRData {
private float value;
+ public MR_float () {}
+
public MR_float ( float x ) { value = x; }
public MR_float ( double x ) { value = (float)x; }
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_int.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_int.java b/core/src/main/java/org/apache/mrql/MR_int.java
index 6586fe0..37e5d9e 100644
--- a/core/src/main/java/org/apache/mrql/MR_int.java
+++ b/core/src/main/java/org/apache/mrql/MR_int.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.fs.*;
final public class MR_int extends MRData {
private int value;
+ public MR_int () {}
+
public MR_int ( int i ) { value = i; }
public void materializeAll () {};
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_long.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_long.java b/core/src/main/java/org/apache/mrql/MR_long.java
index 2078048..e16f2e7 100644
--- a/core/src/main/java/org/apache/mrql/MR_long.java
+++ b/core/src/main/java/org/apache/mrql/MR_long.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
final public class MR_long extends MRData {
private long value;
+ public MR_long () {}
+
public MR_long ( long i ) { value = i; }
public void materializeAll () {};
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_short.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_short.java b/core/src/main/java/org/apache/mrql/MR_short.java
index c50e124..b6e34a8 100644
--- a/core/src/main/java/org/apache/mrql/MR_short.java
+++ b/core/src/main/java/org/apache/mrql/MR_short.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
final public class MR_short extends MRData {
private short value;
+ public MR_short () {}
+
public MR_short ( short i ) { value = i; }
public void materializeAll () {};
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/MR_string.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/MR_string.java b/core/src/main/java/org/apache/mrql/MR_string.java
index a914cca..54cdd59 100644
--- a/core/src/main/java/org/apache/mrql/MR_string.java
+++ b/core/src/main/java/org/apache/mrql/MR_string.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.*;
final public class MR_string extends MRData {
private String value;
+ public MR_string () {}
+
public MR_string ( String s ) { value = s; }
public void materializeAll () {};
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/Main.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/Main.java b/core/src/main/java/org/apache/mrql/Main.java
index b5da0f4..5607a19 100644
--- a/core/src/main/java/org/apache/mrql/Main.java
+++ b/core/src/main/java/org/apache/mrql/Main.java
@@ -61,6 +61,8 @@ final public class Main {
Evaluator.evaluator = (Evaluator)Class.forName("org.apache.mrql.BSPEvaluator").newInstance();
else if (Config.spark_mode)
Evaluator.evaluator = (Evaluator)Class.forName("org.apache.mrql.SparkEvaluator").newInstance();
+ else if (Config.flink_mode)
+ Evaluator.evaluator = (Evaluator)Class.forName("org.apache.mrql.FlinkEvaluator").newInstance();
else // when Config.map_reduce_mode but also the default
Evaluator.evaluator = (Evaluator)Class.forName("org.apache.mrql.MapReduceEvaluator").newInstance();
}
@@ -77,12 +79,15 @@ final public class Main {
public static void main ( String[] args ) throws Exception {
Config.hadoop_mode = false;
+ if (args.length == 2 && args[0].equals("args")) // needed for mrql.flink script
+ args = args[1].substring(1).split("!");
for ( String arg: args ) {
Config.hadoop_mode |= arg.equals("-local") || arg.equals("-dist");
Config.bsp_mode |= arg.equals("-bsp");
Config.spark_mode |= arg.equals("-spark");
+ Config.flink_mode |= arg.equals("-flink");
};
- Config.map_reduce_mode = !Config.bsp_mode && !Config.spark_mode;
+ Config.map_reduce_mode = !Config.bsp_mode && !Config.spark_mode && !Config.flink_mode;
initialize_evaluator();
if (Config.hadoop_mode) {
conf = Evaluator.evaluator.new_configuration();
@@ -110,6 +115,8 @@ final public class Main {
System.out.print("distributed ");
if (Config.spark_mode)
System.out.println("Spark mode using "+Config.nodes+" tasks)");
+ else if (Config.flink_mode)
+ System.out.println("Flink mode using "+Config.nodes+" tasks)");
else if (Config.bsp_mode)
System.out.println("Hama BSP mode over "+Config.nodes+" BSP tasks)");
else if (Config.nodes > 0)
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/ParsedDataSource.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/ParsedDataSource.java b/core/src/main/java/org/apache/mrql/ParsedDataSource.java
index e000405..5b5a06d 100644
--- a/core/src/main/java/org/apache/mrql/ParsedDataSource.java
+++ b/core/src/main/java/org/apache/mrql/ParsedDataSource.java
@@ -28,6 +28,8 @@ public class ParsedDataSource extends DataSource {
public Class<? extends Parser> parser;
public Trees args;
+ ParsedDataSource () {}
+
ParsedDataSource ( int source_num,
String path,
Class<? extends Parser> parser,
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/PlanGeneration.gen
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/PlanGeneration.gen b/core/src/main/java/org/apache/mrql/PlanGeneration.gen
index fe1a1d9..72200fc 100644
--- a/core/src/main/java/org/apache/mrql/PlanGeneration.gen
+++ b/core/src/main/java/org/apache/mrql/PlanGeneration.gen
@@ -672,6 +672,12 @@ final public class PlanGeneration extends AlgebraicOptimization {
match plan {
case MapReduce2(`mx,`my,`r,`x,`y,`o):
return physical_plan(#<MapAggregateReduce2(`mx,`my,`r,null,null,`x,`y,`o)>);
+ case MapAggregateReduce2(`mx,`my,lambda(`v,`b),`acc,`zero,`x,`y,false):
+ if (!Config.flink_mode)
+ fail;
+ return #<MapAggregateReduce2(`mx,`my,lambda(`v,`b),`acc,`zero,
+ `(physical_plan(x)),
+ `(physical_plan(y)),false)>;
// convert a reduce-side join to a fragment-replicate join, if either of the join
// inputs is small to fit in memory (dataset sizes are extracted from file statistics)
case MapAggregateReduce2(`mx,`my,
@@ -750,6 +756,8 @@ final public class PlanGeneration extends AlgebraicOptimization {
case CrossProduct(`mx,`my,`r,`x,`y):
return physical_plan(#<CrossAggregateProduct(`mx,`my,`r,null,null,`x,`y)>);
case CrossAggregateProduct(`mx,`my,lambda(`v,`b),`acc,`zero,`x,`y):
+ if (Config.flink_mode)
+ fail;
Tree X = new_var();
Tree Y = new_var();
Tree nv = new_var();
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/SystemFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/SystemFunctions.java b/core/src/main/java/org/apache/mrql/SystemFunctions.java
index 5c0ed4b..b7385fb 100644
--- a/core/src/main/java/org/apache/mrql/SystemFunctions.java
+++ b/core/src/main/java/org/apache/mrql/SystemFunctions.java
@@ -361,10 +361,23 @@ final public class SystemFunctions {
}
/** used in avg */
- public static MR_double avg_value ( MRData t ) {
- MR_double sum = (MR_double)((Tuple)t).first();
- MR_long count = (MR_long)((Tuple)t).second();
- return new MR_double(sum.get()/count.get());
+ public static MR_double avg_value ( MRData value ) {
+ Tuple t = (Tuple)value;
+ double sum = 0.0;
+ if (t.first() instanceof MR_double)
+ sum = ((MR_double)t.first()).get();
+ else if (t.first() instanceof MR_int)
+ sum = ((MR_int)t.first()).get();
+ else if (t.first() instanceof MR_long)
+ sum = ((MR_long)t.first()).get();
+ else if (t.first() instanceof MR_float)
+ sum = ((MR_float)t.first()).get();
+ long count = 0;
+ if (t.second() instanceof MR_long)
+ count = ((MR_long)t.second()).get();
+ else if (t.second() instanceof MR_int)
+ count = ((MR_int)t.second()).get();
+ return new MR_double(sum/count);
}
public static MR_string text ( Union node ) {
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/Tuple.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/Tuple.java b/core/src/main/java/org/apache/mrql/Tuple.java
index bced3dd..13b0eeb 100644
--- a/core/src/main/java/org/apache/mrql/Tuple.java
+++ b/core/src/main/java/org/apache/mrql/Tuple.java
@@ -28,6 +28,8 @@ final public class Tuple extends MRData {
MRData[] tuple;
+ public Tuple () { tuple = new MRData[0]; }
+
public Tuple ( int size ) {
tuple = new MRData[size];
}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/Union.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/Union.java b/core/src/main/java/org/apache/mrql/Union.java
index 540b6d9..5d92f32 100644
--- a/core/src/main/java/org/apache/mrql/Union.java
+++ b/core/src/main/java/org/apache/mrql/Union.java
@@ -29,6 +29,8 @@ public class Union extends MRData {
private byte tag;
private MRData value;
+ public Union () {}
+
public Union ( byte tag, MRData value ) {
this.tag = tag;
this.value = value;
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/main/java/org/apache/mrql/XMLParser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/mrql/XMLParser.java b/core/src/main/java/org/apache/mrql/XMLParser.java
index a25acbd..f55faba 100644
--- a/core/src/main/java/org/apache/mrql/XMLParser.java
+++ b/core/src/main/java/org/apache/mrql/XMLParser.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.conf.Configuration;
/** An XML parser */
-final public class XMLParser implements Parser {
+public class XMLParser implements Parser {
String[] tags; // split tags
Tree xpath; // XPath query for fragmentation
XMLSplitter splitter;
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/core/src/test/java/org/apache/mrql/QueryTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/mrql/QueryTest.java b/core/src/test/java/org/apache/mrql/QueryTest.java
index b0257dd..a675c9d 100644
--- a/core/src/test/java/org/apache/mrql/QueryTest.java
+++ b/core/src/test/java/org/apache/mrql/QueryTest.java
@@ -49,7 +49,7 @@ public abstract class QueryTest extends TestCase {
resultDir = new File(TEST_RESULT_DIR);
resultDir.mkdirs();
- if(evaluator==null)
+ // if(evaluator==null) // the spark evaluator needs to be recreated
evaluator = createEvaluator();
Translator.global_reset();
for ( Enumeration en = LogManager.getCurrentLoggers(); en.hasMoreElements(); )
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/pom.xml
----------------------------------------------------------------------
diff --git a/flink/pom.xml b/flink/pom.xml
new file mode 100644
index 0000000..a76cda7
--- /dev/null
+++ b/flink/pom.xml
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.mrql</groupId>
+ <artifactId>mrql-flink</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache MRQL Flink mode</name>
+ <description>Apache MRQL evaluation in Flink mode on Apache Flink</description>
+ <url>http://mrql.incubator.apache.org/</url>
+ <inceptionYear>2013</inceptionYear>
+
+ <parent>
+ <groupId>org.apache.mrql</groupId>
+ <artifactId>mrql-parent</artifactId>
+ <version>0.9.4-incubating-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.mrql</groupId>
+ <artifactId>mrql-gen</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mrql</groupId>
+ <artifactId>mrql-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mrql</groupId>
+ <artifactId>mrql-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals><goal>add-source</goal></goals>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.mrql:*</include>
+ </includes>
+ <excludes>
+ <exclude>org.apache.mrql:mrql-gen</exclude>
+ <exclude>org.apache.mrql:mrql-core</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <finalName>mrql-flink-${project.version}</finalName>
+ <outputDirectory>${project.parent.basedir}/lib</outputDirectory>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <mkdir dir="${project.build.directory}/generated-sources/org/apache/mrql" />
+ <property name="compile_classpath" refid="maven.compile.classpath" />
+ <fileset id="mr.gen.path" dir="src/main/java/org/apache/mrql" includes="*.gen" />
+ <pathconvert pathsep=" " property="mr.gen.files" refid="mr.gen.path" />
+ <java classname="org.apache.mrql.gen.Main" classpath="../lib/mrql-gen-${project.version}.jar:${compile_classpath}">
+ <arg line="${mr.gen.files}" />
+ <arg line="-o" />
+ <arg file="${project.build.directory}/generated-sources/org/apache/mrql" />
+ </java>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FData.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FData.java b/flink/src/main/java/org/apache/mrql/FData.java
new file mode 100644
index 0000000..7cb288c
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FData.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mrql;
+
+import java.io.IOException;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.types.CopyableValue;
+
+
+final public class FData implements CopyableValue<FData>, Comparable<FData> {
+ MRData data;
+
+ public FData () {}
+
+ public FData ( MRData d ) { data = d; }
+
+ public MRData data () { return data; }
+
+ @Override
+ public void read ( DataInputView in ) throws IOException {
+ data = MRContainer.read(in);
+ }
+
+ @Override
+ public void write ( DataOutputView out ) throws IOException {
+ data.write(out);
+ }
+
+ @Override
+ public int getBinaryLength () { return -1; }
+
+ @Override
+ public void copyTo ( FData target ) {
+ target.data = data;
+ }
+
+ @Override
+ public void copy ( DataInputView source, DataOutputView target ) throws IOException {
+ data = MRContainer.read(source);
+ data.write(target);
+ }
+
+ @Override
+ public int compareTo ( FData x ) { return data.compareTo(x.data); }
+
+ @Override
+ public boolean equals ( Object x ) {
+ return x instanceof FData && data.equals(((FData)x).data);
+ }
+
+ @Override
+ public int hashCode () { return data.hashCode(); }
+
+ @Override
+ public String toString () { return data.toString(); }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FDataContainer.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FDataContainer.java b/flink/src/main/java/org/apache/mrql/FDataContainer.java
new file mode 100644
index 0000000..07eaa22
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FDataContainer.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mrql;
+
+import java.io.IOException;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.core.io.IOReadableWritable;
+
+final public class FDataContainer extends MRContainer implements IOReadableWritable, Comparable<MRContainer> {
+
+ FDataContainer ( MRData d ) { super(d); }
+
+ @Override
+ public void read ( DataInputView in ) throws IOException {
+ data = super.read(in);
+ }
+
+ @Override
+ public void write ( DataOutputView out ) throws IOException {
+ super.write(out);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkBinaryInputFormat.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkBinaryInputFormat.java b/flink/src/main/java/org/apache/mrql/FlinkBinaryInputFormat.java
new file mode 100644
index 0000000..5969c1e
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkBinaryInputFormat.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.io.*;
+import org.apache.flink.api.common.io.BinaryInputFormat;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.api.common.io.FileInputFormat;
+
+
+/** Input format for sequence files */
+final public class FlinkBinaryInputFormat extends FlinkMRQLFileInputFormat {
+ public FlinkBinaryInputFormat () {}
+
+ public static final class FDataInputFormat extends BinaryInputFormat<FData> {
+ @Override
+ protected FData deserialize ( FData reuse, DataInputView in ) throws IOException {
+ reuse.read(in);
+ return reuse;
+ }
+ }
+
+ /** the Flink input format for this input */
+ public FileInputFormat<FData> inputFormat ( String path ) {
+ FDataInputFormat sf = new FDataInputFormat();
+ sf.setFilePath(path.toString());
+ return sf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkDataSource.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkDataSource.java b/flink/src/main/java/org/apache/mrql/FlinkDataSource.java
new file mode 100644
index 0000000..6c98ade
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkDataSource.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.mrql.gen.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.io.IOException;
+import java.io.Serializable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.common.accumulators.Accumulator;
+import org.apache.flink.api.common.accumulators.LongCounter;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.Collector;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.core.fs.FileInputSplit;
+
+
+/** A data source for storing a Flink result (a Flink DataSet ) */
+final public class FlinkDataSource extends DataSource implements Serializable {
+ public DataSet<FData> data_set;
+
+ /** Used in MRQL total aggregations in FlinkEvaluator.aggregate */
+ public static final class Reducer implements Accumulator<FData,String> {
+ public FData accumulated_value;
+ public FData zero;
+ public String acc_fnc;
+ public String merge_fnc;
+ transient Function acc;
+ transient Function merge;
+
+ public Reducer () {}
+
+ public Reducer ( MRData zero, String acc_fnc, String merge_fnc ) {
+ this.zero = new FData(zero);
+ accumulated_value = this.zero;
+ this.acc_fnc = acc_fnc;
+ this.merge_fnc = merge_fnc;
+ }
+
+ public void add ( FData value ) {
+ if (acc == null)
+ try {
+ acc = Interpreter.evalF(Tree.parse(acc_fnc),null);
+ } catch (Exception ex) {
+ throw new Error(ex);
+ };
+ accumulated_value = new FData(acc.eval(new Tuple(accumulated_value.data(),value.data())));
+ }
+
+ public String getLocalValue () {
+ // due to a Flink bug, we cannot return a custom object (FData) from an accumulator
+ return accumulated_value.data().toString();
+ }
+
+ public void resetLocal () {
+ accumulated_value = zero;
+ }
+
+ public void merge ( Accumulator<FData,String> other ) {
+ if (merge == null)
+ try {
+ merge = Interpreter.evalF(Tree.parse(merge_fnc),null);
+ } catch (Exception ex) {
+ throw new Error(ex);
+ };
+ accumulated_value = new FData(merge.eval(new Tuple(accumulated_value.data(),
+ ((Reducer)other).accumulated_value.data())));
+ }
+
+ public void write ( DataOutputView out ) throws IOException {
+ accumulated_value.write(out);
+ zero.write(out);
+ out.writeUTF(acc_fnc);
+ out.writeUTF(merge_fnc);
+ }
+
+ public void read ( DataInputView in ) throws IOException {
+ accumulated_value = new FData();
+ zero = new FData();
+ accumulated_value.read(in);
+ zero.read(in);
+ acc_fnc = in.readUTF();
+ merge_fnc = in.readUTF();
+ }
+ }
+
+ FlinkDataSource ( DataSet<FData> d, String path, boolean to_be_merged ) {
+ super();
+ this.data_set = d;
+ this.path = path;
+ this.inputFormat = FlinkBinaryInputFormat.class;
+ this.to_be_merged = to_be_merged;
+ }
+
+ public static final class size_mapper extends RichFlatMapFunction<FData,FData> {
+ private LongCounter c = new LongCounter();
+
+ @Override
+ public void open ( org.apache.flink.configuration.Configuration parameters ) throws Exception {
+ super.open(parameters);
+ getRuntimeContext().addAccumulator("count",c);
+ }
+
+ @Override
+ public void flatMap ( FData value, Collector<FData> out ) throws Exception {
+ c.add(1L);
+ }
+ }
+
+ /** not used */
+ @Override
+ public long size ( Configuration conf ) {
+ return Long.MAX_VALUE;
+ }
+
+ /** return the first num values */
+ @Override
+ public List<MRData> take ( int num ) {
+ try {
+ // used for printing query results; currently, terribly inefficient
+ List<MRData> a = new ArrayList<MRData>();
+ final FData container = new FData();
+ if (!to_be_merged) {
+ FileInputFormat<FData> sf = (new FlinkBinaryInputFormat()).inputFormat(path);
+ FileInputSplit[] splits = sf.createInputSplits(1);
+ long i = 0;
+ for ( FileInputSplit split: splits ) {
+ sf.open(split);
+ while (!sf.reachedEnd() && (num < 0 || i < num)) {
+ a.add(sf.nextRecord(container).data());
+ i++;
+ };
+ if (num > 0 && i >= num)
+ break;
+ };
+ sf.close();
+ return a;
+ } else { // needs merging since splits came out of sorting
+ FlinkBinaryInputFormat ifbif = new FlinkBinaryInputFormat();
+ FileInputFormat<FData> sf = ifbif.inputFormat(path);
+ FileInputSplit[] splits = sf.createInputSplits(1);
+ FileInputFormat<FData>[] s = new FlinkBinaryInputFormat.FDataInputFormat[splits.length];
+ MRData[] key = new MRData[splits.length];
+ MRData[] value = new MRData[splits.length];
+ int min = 0;
+ int i = 0;
+ for ( FileInputSplit split: splits ) {
+ s[i] = ifbif.inputFormat(path);
+ s[i].open(split);
+ i++;
+ };
+ for ( int j = 0; j < splits.length; j++ ) {
+ if (!s[j].reachedEnd()) {
+ Tuple v = (Tuple)s[j].nextRecord(container).data();
+ key[j] = v.first();
+ value[j] = v.second();
+ }
+ };
+ long c = 0;
+ while ( (num < 0 || c < num) && min >= 0) {
+ min = -1;
+ for ( int j = 0; j < splits.length; j++ )
+ if (key[j] != null && (min < 0 || key[j].compareTo(key[min]) < 0))
+ min = j;
+ if (min >= 0) {
+ a.add(value[min]);
+ if (!s[min].reachedEnd()) {
+ Tuple v = (Tuple)s[min].nextRecord(container).data();
+ key[min] = v.first();
+ value[min] = v.second();
+ } else key[min] = null;
+ };
+ c++;
+ };
+ for ( int j = 0; j < splits.length; j++ )
+ s[j].close();
+ return a;
+ }
+ } catch (IOException ex) {
+ throw new Error(ex);
+ }
+ }
+
+ /** merge the splits of a flink data source that have already been sorted */
+ public Bag merge () {
+ try {
+ FlinkBinaryInputFormat ifbif = new FlinkBinaryInputFormat();
+ FileInputFormat<FData> sf = ifbif.inputFormat(path);
+ FileInputSplit[] splits = sf.createInputSplits(1);
+ final int n = splits.length;
+ final FileInputFormat<FData>[] s = new FlinkBinaryInputFormat.FDataInputFormat[n];
+ int i = 0;
+ for ( FileInputSplit split: splits ) {
+ s[i] = ifbif.inputFormat(path);
+ s[i].open(split);
+ i++;
+ };
+ return new Bag(new BagIterator () {
+ int min = 0;
+ boolean first = true;
+ MRData[] keys = new MRData[n];
+ MRData[] values = new MRData[n];
+ MRData key, value;
+ final FData container = new FData();
+ public boolean hasNext () {
+ try {
+ if (first) {
+ first = false;
+ for ( int i = 0; i < n; i++ ) {
+ if (!s[i].reachedEnd()) {
+ Tuple v = (Tuple)s[i].nextRecord(container).data();
+ keys[i] = v.first();
+ values[i] = v.second();
+ }
+ }
+ };
+ min = -1;
+ for ( int i = 0; i < n; i++ )
+ if (keys[i] != null && (min < 0 || keys[i].compareTo(keys[min]) < 0))
+ min = i;
+ if (min >= 0) {
+ key = keys[min];
+ value = values[min];
+ if (!s[min].reachedEnd()) {
+ Tuple v = (Tuple)s[min].nextRecord(container).data();
+ keys[min] = v.first();
+ values[min] = v.second();
+ } else keys[min] = null;
+ return true;
+ } else {
+ for ( int i = 0; i < n; i++ )
+ s[i].close();
+ return false;
+ }
+ } catch (IOException e) {
+ throw new Error("Cannot merge values from an intermediate result: "+e);
+ }
+ }
+ public MRData next () { return value; }
+ });
+ } catch (IOException e) {
+ throw new Error("Cannot merge values from an intermediate result: "+e);
+ }
+ }
+
+ /** Used in MRQL total aggregations in FlinkEvaluator.aggregate */
+ public static final class reduce_mapper extends RichFlatMapFunction<FData,FData> {
+ Reducer r;
+
+ reduce_mapper ( MRData zero, String acc, String merge ) {
+ r = new Reducer(zero,acc,merge);
+ }
+
+ @Override
+ public void open ( org.apache.flink.configuration.Configuration parameters ) throws Exception {
+ super.open(parameters);
+ getRuntimeContext().addAccumulator("reducer",r);
+ }
+
+ @Override
+ public void flatMap ( FData value, Collector<FData> out ) throws Exception {
+ r.add(value);
+ }
+ }
+
+ /** accumulate all dataset values */
+ @Override
+ public MRData reduce ( MRData zero, Function acc ) {
+ throw new Error("Flink reduce operation is not implemented yet");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen b/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
new file mode 100644
index 0000000..140a7f2
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
@@ -0,0 +1,691 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.io.PrintStream;
+import java.net.URI;
+import java.net.URL;
+import org.apache.mrql.gen.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.flink.util.Collector;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.RemoteEnvironment;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.IterativeDataSet;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.io.LocalCollectionOutputFormat;
+import org.apache.flink.api.java.functions.RichMapFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichCoGroupFunction;
+import org.apache.flink.api.java.functions.RichCrossFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
+import org.apache.flink.api.java.functions.RichFilterFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.GroupReduceOperator;
+
+
+/** Evaluates physical plans in Apache Flink mode */
+final public class FlinkEvaluator extends Evaluator {
+ final static URL flink_jar = FlinkEvaluator.class.getProtectionDomain().getCodeSource().getLocation();
+ public static ExecutionEnvironment flink_env;
+
+ /** initialize the Flink evaluator */
+ final public void init ( Configuration conf ) {
+ DataSource.parserDirectory = new DataSource.ParserDirectory();
+ DataSource.parserDirectory.put("xml",FlinkXMLParser.class);
+ DataSource.parserDirectory.put("json",FlinkJsonParser.class);
+ DataSource.parserDirectory.put("line",FlinkLineParser.class);
+ if (Config.local_mode) {
+ flink_env = ExecutionEnvironment.createLocalEnvironment(Config.nodes);
+ // curently, the compiler desn't work in local mode
+ Config.compile_functional_arguments = false;
+ Config.write(Plan.conf);
+ } else flink_env = ExecutionEnvironment.getExecutionEnvironment();
+ }
+
+ /** shutdown the Flink evaluator */
+ final public void shutdown ( Configuration conf ) {
+ }
+
+ final public void initialize_query () {
+ try {
+ if (!Config.local_mode) {
+ Plan.distribute_compiled_arguments(Plan.conf);
+ String master_node = System.getenv("FLINK_MASTER");
+ if (Config.compile_functional_arguments)
+ flink_env = ExecutionEnvironment.createRemoteEnvironment(master_node,6123,
+ flink_jar.toURI().getPath(),Plan.conf.get("mrql.jar.path"));
+ else flink_env = ExecutionEnvironment.createRemoteEnvironment(master_node,6123,
+ flink_jar.toURI().getPath());
+ flink_env.setDefaultLocalParallelism(Config.nodes);
+ };
+ } catch (Exception ex) {
+ throw new Error(ex);
+ }
+ }
+
+ final public Configuration new_configuration () {
+ return new Configuration();
+ }
+
+ /** return the FileInputFormat for parsed files (CSV, XML, JSON, etc) */
+ final public Class<? extends MRQLFileInputFormat> parsedInputFormat () {
+ return FlinkParsedInputFormat.class;
+ }
+
+ /** return the FileInputFormat for binary files */
+ final public Class<? extends MRQLFileInputFormat> binaryInputFormat () {
+ return FlinkBinaryInputFormat.class;
+ }
+
+ /** return the FileInputFormat for data generator files */
+ final public Class<? extends MRQLFileInputFormat> generatorInputFormat () {
+ return FlinkGeneratorInputFormat.class;
+ }
+
+ /** returns the absolute path relative to the directory that contains the MRQL executable */
+ private String absolute_path ( String path) {
+ try {
+ URI uri = new URI(path);
+ if (uri.getScheme() == null)
+ if (uri.toString().startsWith("/"))
+ uri = new URI("file:"+uri);
+ else uri = new URI("file:"+System.getProperty("user.dir")+"/"+uri);
+ uri = uri.normalize();
+ return uri.toString();
+ } catch (Exception ex) {
+ throw new Error("Wrong pathname: "+path);
+ }
+ }
+
+ /** dump MRQL data into a Flink binary file instead of a Hadoop Sequence file */
+ @Override
+ public void dump ( String file, Tree type, MRData data ) throws Exception {
+ Path path = new Path(absolute_path(file));
+ FileSystem fs = path.getFileSystem();
+ PrintStream ftp = new PrintStream(fs.create(path.suffix(".type"),true));
+ ftp.print("2@"+type.toString()+"\n");
+ ftp.close();
+ if (data instanceof MR_dataset)
+ data = Plan.collect(((MR_dataset)data).dataset());
+ DataSet<Long> d = flink_env.generateSequence(0L,0L); // need just one record
+ DataSet<FData> res = (data instanceof Bag)
+ ? d.flatMap(new bag_dumper(new FData(data)))
+ : d.map(new value_dumper(new FData(data)));
+ res.write(new FlinkOutputFormat(),path.toString(),
+ org.apache.flink.core.fs.FileSystem.WriteMode.OVERWRITE);
+ flink_env.execute("MRQL dump");
+ }
+
+ public static final class bag_dumper extends RichFlatMapFunction<Long,FData> {
+ final FData container = new FData(new MR_int(0));
+ final FData value;
+
+ bag_dumper ( FData value ) { this.value = value; }
+
+ @Override
+ public void flatMap ( Long ignore, Collector<FData> out ) throws Exception {
+ for ( MRData e: (Bag)value.data() ) {
+ container.data = e;
+ out.collect(container);
+ }
+ }
+ }
+
+ public static final class value_dumper extends RichMapFunction<Long,FData> {
+ final FData value;
+
+ value_dumper ( FData value ) { this.value = value; }
+
+ @Override
+ public FData map ( Long ignore ) {
+ return value;
+ }
+ }
+
+ private static MRData aggregate ( DataSet<FData> data_set, MRData zero, Tree acc_fnc, Tree merge_fnc ) {
+ DataSet<FData> d = data_set.flatMap(new FlinkDataSource.reduce_mapper(zero,acc_fnc.toString(),
+ merge_fnc.toString()));
+ d.print(); // needs a datasink (prints the aggregation value)
+ try {
+ // due to a Flink bug, we cannot return a custom object (FData) from an accumulator
+ String value = FlinkEvaluator.flink_env.execute("MRQL aggregator").getAccumulatorResult("reducer");
+ return MRQL.query(value); // parse the value using the MRQL query parser (ugly)
+ } catch (Exception ex) {
+ throw new Error("Evaluation error during aggregation: "+ex);
+ }
+ }
+
+ /** The Aggregate physical operator
+ * @param merge the accumulator function from (T,T) to T
+ * @param zero the zero element of type T
+ * @param plan the plan that constructs the dataset that contains the bag of values {T}
+ * @param env contains bindings fro variables to values (MRData)
+ * @return the aggregation result of type T
+ */
+ final public MRData aggregate ( Tree merge,
+ Tree zero,
+ Tree plan,
+ Environment env ) throws Exception {
+ MRData z = evalE(zero,env);
+ match plan {
+ case AggregateMap(`m,`acc,_,`s):
+ return aggregate(evalD(#<cMap(`m,`s)>,env),z,acc,merge);
+ case MapAggregateReduce(`m,`r,`acc,_,`s,`o):
+ if (acc.equals(#<null>))
+ fail;
+ return aggregate(evalD(#<MapReduce(`m,`r,`s,`o)>,env),z,acc,merge);
+ case CrossAggregateProduct(`mx,`my,`r,`acc,_,`x,`y):
+ if (acc.equals(#<null>))
+ fail;
+ return aggregate(evalD(#<CrossProduct(`mx,`my,`r,`x,`y)>,env),z,acc,merge);
+ case MapAggregateReduce2(`mx,`my,`r,`acc,_,`x,`y,`o):
+ if (acc.equals(#<null>))
+ fail;
+ return aggregate(evalD(#<MapReduce2(`mx,`my,`r,`x,`y,`o)>,env),z,acc,merge);
+ case MapAggregateJoin(`mx,`my,`r,`acc,_,`x,`y):
+ if (acc.equals(#<null>))
+ fail;
+ return aggregate(evalD(#<MapJoin(`mx,`my,`r,`x,`y)>,env),z,acc,merge);
+ };
+ throw new Error("Unrecognized aggregation: "+plan);
+ }
+
+ /** Evaluate a loop a fixed number of times. Doesn't use Flink iterations */
+ final public Tuple loop ( Tree e, Environment env ) throws Exception {
+ match e {
+ case Loop(lambda(tuple(...vs),tuple(...bs)),tuple(...ss),`num):
+ final int limit = ((MR_int)evalE(num,env)).get();
+ MR_flink[] s = new MR_flink[vs.length()];
+ String[] path = new String[vs.length()];
+ for ( int i = 0; i < vs.length(); i++ )
+ s[i] = new MR_flink(eval(ss.nth(i),env));
+ for ( int n = 0; n < limit; n++ ) {
+ Environment nenv = env;
+ for ( int i = 0; i < vs.length(); i ++ )
+ nenv = new Environment(vs.nth(i).toString(),s[i],nenv);
+ for ( int i = 0; i < vs.length(); i ++ )
+ s[i] = new MR_flink(eval(bs.nth(i),nenv));
+ };
+ for ( int i = 0; i < vs.length(); i++ ) {
+ path[i] = absolute_path(Plan.new_path(Plan.conf));
+ s[i].flink().write(new FlinkOutputFormat(),path[i]);
+ };
+ flink_env.execute("MRQL loop");
+ final FlinkBinaryInputFormat bif = new FlinkBinaryInputFormat();
+ Tuple t = new Tuple(vs.length());
+ for ( int i = 0; i < vs.length(); i++ )
+ t.set(i,bif.materialize(new org.apache.hadoop.fs.Path(path[i])));
+ return t;
+ };
+ throw new Error("Wrong Loop format: "+e);
+ }
+
+ private static boolean needs_sorting ( Tree e ) {
+ match e {
+ case MapReduce(`m,`r,`s,true): return true;
+ case MapCombineReduce(`m,`c,`r,`s,true): return true;
+ };
+ return false;
+ }
+
+ /** Evaluate an MRQL physical plan and print tracing info
+ * @param e the physical plan
+ * @param env contains bindings fro variables to values (MRData)
+ * @return an MRQL DataSet (stored in HDFS)
+ */
+ final public org.apache.mrql.DataSet eval ( final Tree e,
+ final Environment env,
+ final String counter ) {
+ DataSet<FData> d = eval(e,env);
+ try {
+ String path = absolute_path(Plan.new_path(Plan.conf));
+ d.write(new FlinkOutputFormat(),path);
+ flink_env.execute("MRQL query");
+ return new org.apache.mrql.DataSet(new FlinkDataSource(d,path,needs_sorting(e)),0L,0L);
+ } catch (Exception ex) {
+ throw new Error("Cannot evaluate the query: "+e);
+ }
+ }
+
+ /** Evaluate an MRQL physical plan using Flink and print tracing info
+ * @param e the physical plan
+ * @param env contains bindings from variables to values (MRData)
+ * @return a Flink DataSet
+ */
+ final public DataSet<FData> eval ( final Tree e, final Environment env ) {
+ if (Config.trace_execution) {
+ tab_count += 3;
+ System.out.println(tabs(tab_count)+print_query(e));
+ };
+ final DataSet<FData> res = evalD(e,env);
+ if (Config.trace_execution)
+ try {
+ List<FData> out = new ArrayList<FData>();
+ res.output(new LocalCollectionOutputFormat(out));
+ System.out.println(tabs(tab_count)+"-> "+out);
+ tab_count -= 3;
+ } catch (Exception ex) {
+ throw new Error("Cannot collect the operator output: "+e);
+ };
+ return res;
+ }
+
+ /* convert an MRQL lambda to a Flink Function */
+ private static RichFlatMapFunction<FData,FData> cmap_fnc ( final Tree fnc ) {
+ final Function f = evalF(fnc,null);
+ return new RichFlatMapFunction<FData,FData>() {
+ @Override
+ public void flatMap ( FData value, Collector<FData> out ) throws Exception {
+ for ( MRData e: (Bag)f.eval(value.data()) )
+ out.collect(new FData(e));
+ }
+ };
+ }
+
+ /* convert the stream of pairs to a bag of their second element */
+ public static Bag bag ( Iterable<FData> values ) {
+ Bag s = new Bag();
+ for ( FData value: values )
+ s.add(((Tuple)value.data()).second());
+ return s;
+ }
+
+ /* group-by s and then reduce by reduce_fnc (optional: use combine_fnc) */
+ private static DataSet<FData> groupBy ( DataSet<FData> s, Tree combine_fnc, Tree reduce_fnc ) {
+ final Function combiner = (combine_fnc == null) ? null : evalF(combine_fnc,null);
+ final Function reducer = evalF(reduce_fnc,null);
+ GroupReduceOperator<FData,FData> ro = s.groupBy(new KeySelector<FData,FData>() {
+ @Override
+ public FData getKey ( FData value ) {
+ return new FData(((Tuple)value.data()).first());
+ }
+ }).reduceGroup(new RichGroupReduceFunction<FData,FData>() {
+ @Override
+ public void reduce ( final Iterable<FData> values, Collector<FData> out ) {
+ Bag s = new Bag();
+ MRData key = null;
+ for ( FData value: values ) {
+ Tuple t = (Tuple)value.data();
+ key = t.first();
+ s.add(t.second());
+ };
+ for ( MRData v: (Bag)reducer.eval(new Tuple(key,s)) )
+ out.collect(new FData(v));
+ }
+ @Override
+ public void combine ( Iterable<FData> values, Collector<FData> out ) throws Exception {
+ Bag c = null;
+ MRData key = null;
+ for ( FData v: values )
+ if (c == null) {
+ c = new Bag(((Tuple)v.data()).second());
+ key = ((Tuple)v.data()).first();
+ } else
+ c = (Bag)combiner.eval(new Tuple(key,c.add_element(((Tuple)v.data()).second())));
+ for ( MRData x: c )
+ out.collect(new FData(new Tuple(key,x)));
+ }
+ });
+ ro.setCombinable(combiner != null);
+ return ro;
+ }
+
+ /* group-by and sort s and then reduce by reduce_fnc (optional: use combine_fnc) */
+ private static DataSet<FData> sortBy ( DataSet<FData> s, Tree combine_fnc, Tree reduce_fnc ) {
+ final Function combiner = (combine_fnc == null) ? null : evalF(combine_fnc,null);
+ final Function reducer = evalF(reduce_fnc,null);
+ GroupReduceOperator<Tuple2<FData,FData>,FData> ro
+ = s.map(new RichMapFunction<FData,Tuple2<FData,FData>>() {
+ @Override
+ public Tuple2<FData,FData> map ( FData value ) {
+ Tuple t = (Tuple)value.data();
+ return new Tuple2<FData,FData>(new FData(t.first()),new FData(t.second()));
+ }
+ }).groupBy(0).sortGroup(0,Order.ASCENDING)
+ .reduceGroup(new RichGroupReduceFunction<Tuple2<FData,FData>,FData>() {
+ @Override
+ public void reduce ( final Iterable<Tuple2<FData,FData>> values, Collector<FData> out ) {
+ Bag s = new Bag();
+ MRData key = null;
+ for ( Tuple2<FData,FData> value: values ) {
+ key = value.f0.data();
+ s.add(value.f1.data());
+ };
+ for ( MRData v: (Bag)reducer.eval(new Tuple(key,s)) )
+ out.collect(new FData(new Tuple(key,v)));
+ }
+ @Override
+ public void combine ( Iterable<Tuple2<FData,FData>> values,
+ Collector<Tuple2<FData,FData>> out ) throws Exception {
+ Bag c = null;
+ FData key = null;
+ for ( Tuple2<FData,FData> v: values )
+ if (c == null) {
+ c = new Bag(v.f1.data());
+ key = v.f0;
+ } else
+ c = (Bag)combiner.eval(new Tuple(key.data(),c.add_element(v.f0.data())));
+ for ( MRData x: c )
+ out.collect(new Tuple2<FData,FData>(key,new FData(x)));
+ }
+ });
+ ro.setCombinable(combiner != null);
+ return ro;
+ }
+
+ public static final class join_key implements KeySelector<FData,FData> {
+ @Override
+ public FData getKey ( FData value ) {
+ return new FData(((Tuple)value.data()).first());
+ }
+ }
+
+ public static final class join_reducer extends RichCoGroupFunction<FData,FData,FData> {
+ final Function f;
+
+ join_reducer ( Tree fnc ) {
+ f = evalF(fnc,null);
+ }
+
+ @Override
+ public void coGroup ( Iterable<FData> xs, Iterable<FData> ys, Collector<FData> out ) {
+ for ( MRData v: (Bag)f.eval(new Tuple(bag(xs),bag(ys))) )
+ out.collect(new FData(v));
+ }
+ }
+
+ public static final class cross_reducer extends RichCrossFunction<FData,FData,FData> {
+ final Function f;
+
+ cross_reducer ( Tree fnc ) {
+ f = evalF(fnc,null);
+ }
+
+ @Override
+ public FData cross ( FData x, FData y ) throws Exception {
+ return new FData(f.eval(new Tuple(x.data(),y.data())));
+ }
+ }
+
+ public static final class mapjoin_reducer extends RichJoinFunction<FData,FData,FData> {
+ final Function f;
+
+ mapjoin_reducer ( Tree fnc ) {
+ f = evalF(fnc,null);
+ }
+
+ @Override
+ public FData join ( FData x, FData y ) throws Exception {
+ return new FData(f.eval(new Tuple(x.data(),y.data())));
+ }
+ }
+
+ /** restore the global bindings at the client node */
+ public static final class restore_global_functions extends RichMapFunction<FData,FData> {
+ Environment client_env;
+
+ restore_global_functions () {
+ client_env = Interpreter.global_env;
+ }
+
+ @Override
+ public void open ( org.apache.flink.configuration.Configuration parameters ) throws Exception {
+ Interpreter.set_global_bindings(client_env);
+ }
+
+ @Override
+ public FData map ( FData value ) throws Exception { return value; }
+ }
+
+ public static final class generator_mapper extends RichMapFunction<Long,FData> {
+ @Override
+ public FData map ( Long value ) throws Exception {
+ return new FData(new MR_long(value));
+ }
+ }
+
+ public static final class parsed_mapper extends RichFlatMapFunction<String,FData> {
+ final Parser parser;
+
+ parsed_mapper ( Class<? extends Parser> parser_class ) throws Exception {
+ parser = parser_class.newInstance();
+ }
+
+ @Override
+ public void flatMap ( String value, Collector<FData> out ) throws Exception {
+ for ( MRData e: parser.parse(value) )
+ out.collect(new FData(e));
+ }
+ }
+
+ public static final class group_join_left extends RichFlatMapFunction<FData,FData> {
+ final MRData one = new MR_byte(1);
+ final Function fgx;
+ final int m, n;
+
+ group_join_left ( Tree gx, int m, int n ) {
+ this.m = m;
+ this.n = n;
+ fgx = evalF(gx,null);
+ }
+
+ @Override
+ public void flatMap ( FData value, Collector<FData> out ) throws Exception {
+ for ( int i = 0; i < n; i++ ) {
+ final MRData key = new MR_int((fgx.eval(value.data()).hashCode() % m)+m*i);
+ out.collect(new FData(new Tuple(key,new Tuple(one,value.data()))));
+ }
+ }
+ }
+
+ public static final class group_join_right extends RichFlatMapFunction<FData,FData> {
+ final MRData two = new MR_byte(2);
+ final Function fgy;
+ final int m, n;
+
+ group_join_right ( Tree gy, int m, int n ) {
+ this.m = m;
+ this.n = n;
+ fgy = evalF(gy,null);
+ }
+
+ @Override
+ public void flatMap ( FData value, Collector<FData> out ) throws Exception {
+ for ( int j = 0; j < m; j++ ) {
+ final MRData key = new MR_int((fgy.eval(value.data()).hashCode() % n)*m+j);
+ out.collect(new FData(new Tuple(key,new Tuple(two,value.data()))));
+ }
+ }
+ }
+
+ public static final class group_join_reducer extends RichGroupReduceFunction<FData,FData> {
+ final Function fkx, fky, fgx, fgy, fc, fr;
+ final MRData z;
+
+ group_join_reducer ( Tree kx, Tree ky, Tree gx, Tree gy, Tree c, Tree r, Tree zero ) {
+ fkx = evalF(kx,null); fky = evalF(ky,null); fgx = evalF(gx,null); fgy = evalF(gy,null);
+ fr = evalF(r,null); fc = evalF(c,null); z = evalE(zero,null);
+ }
+
+ @Override
+ public void reduce ( final Iterable<FData> values, Collector<FData> out ) {
+ Bag xb = new Bag();
+ Bag yb = new Bag();
+ for ( FData value: values ) {
+ final Tuple t = (Tuple)value.data();
+ final Tuple p = (Tuple)t.second();
+ if (((MR_byte)p.first()).get() == 1)
+ xb.add(new Tuple(t.first(),p.second()));
+ else yb.add(new Tuple(t.first(),p.second()));
+ }
+ final Bag b = MapReduceAlgebra.mergeGroupByJoin(fkx,fky,fgx,fgy,fc,z,fr,xb,yb);
+ for ( MRData v: b )
+ out.collect(new FData(v));
+ };
+ }
+
+ public static final class repeat_values extends RichMapFunction<FData,FData> {
+ @Override
+ public FData map ( FData value ) throws Exception {
+ return new FData(((Tuple)value.data()).first());
+ }
+ }
+
+ public static final class repeat_filter extends RichFilterFunction<FData> {
+ @Override
+ public boolean filter ( FData value ) throws Exception {
+ return ((MR_bool)(((Tuple)value.data()).second())).get();
+ }
+ }
+
+ /** Evaluate a MRQL physical plan
+ * @param e the physical plan
+ * @param env contains bindings from variables to values (MRData)
+ * @return a Flink DataSet
+ */
+ final private DataSet<FData> evalD ( final Tree e, final Environment env ) {
+ try {
+ match e {
+ case MapAggregateReduce(`m,`r,null,_,`s,`o):
+ return evalD(#<MapReduce(`m,`r,`s,`o)>,env);
+ case CrossAggregateProduct(`mx,`my,`r,null,_,`x,`y):
+ return evalD(#<CrossProduct(`mx,`my,`r,`x,`y)>,env);
+ case MapAggregateReduce2(`mx,`my,`r,null,_,`x,`y,`o):
+ return evalD(#<MapReduce2(`mx,`my,`r,`x,`y,`o)>,env);
+ case MapAggregateJoin(`mx,`my,`r,null,_,`x,`y):
+ return evalD(#<MapJoin(`mx,`my,`r,`x,`y)>,env);
+ case cMap(`f,`s):
+ return eval(s,env).flatMap(cmap_fnc(f));
+ case MapReduce(`m,`r,`s,true):
+ return sortBy(eval(s,env).flatMap(cmap_fnc(m)),null,r);
+ case MapReduce(`m,`r,`s,`o):
+ return groupBy(eval(s,env).flatMap(cmap_fnc(m)),null,r);
+ case MapCombineReduce(`m,`c,`r,`s,true):
+ return sortBy(eval(s,env).flatMap(cmap_fnc(m)),c,r);
+ case MapCombineReduce(`m,`c,`r,`s,`o):
+ return groupBy(eval(s,env).flatMap(cmap_fnc(m)),c,r);
+ case MapCombineReduce2(`mx,`my,`c,`r,`x,`y,`o):
+ return evalD(#<MapReduce2(`mx,`my,`r,`x,`y,`o)>,env);
+ case MapReduce2(`mx,`my,`r,`x,`y,`o):
+ final DataSet<FData> xs = eval(x,env).flatMap(cmap_fnc(mx));
+ final DataSet<FData> ys = eval(y,env).flatMap(cmap_fnc(my));
+ final DataSet<FData> ds = xs.coGroup(ys)
+ .where(new join_key())
+ .equalTo(new join_key())
+ .with(new join_reducer(r));
+ if (o.equals(#<true>))
+ return sortBy(ds,null,#<lambda(x,bag(x))>);
+ else return ds;
+ case MapJoin(`mx,`my,`r,`x,`y):
+ final DataSet<FData> xs = eval(x,env).flatMap(cmap_fnc(mx));
+ final DataSet<FData> ys = eval(y,env).flatMap(cmap_fnc(my));
+ return xs.joinWithTiny(ys)
+ .where(new join_key())
+ .equalTo(new join_key())
+ .with(new mapjoin_reducer(r));
+ case CrossProduct(`mx,`my,`r,`x,`y):
+ final DataSet<FData> xs = eval(x,env).flatMap(cmap_fnc(mx));
+ final DataSet<FData> ys = eval(y,env).flatMap(cmap_fnc(my));
+ return xs.cross(ys).with(new cross_reducer(r))
+ .flatMap(cmap_fnc(#<lambda(x,x)>));
+ case GroupByJoin(`kx,`ky,`gx,`gy,`acc,`zero,`r,`x,`y,`o):
+ final int n = Math.max(2,(int)Math.floor(Math.sqrt(Config.nodes)));
+ final int m = n;
+ // System.err.println("Using a groupBy join on a "+n+"*"+m+" grid of partitions");
+ final DataSet<FData> xs = eval(x,env).flatMap(new group_join_left(gx,m,n));
+ final DataSet<FData> ys = eval(y,env).flatMap(new group_join_right(gx,m,n));
+ return xs.union(ys).groupBy(new join_key())
+ .reduceGroup(new group_join_reducer(kx,ky,gx,gy,acc,r,zero));
+ case Merge(`x,`y):
+ return eval(x,env).union(eval(y,env));
+ case Generator(`min,`max,`size):
+ final long from = ((MR_long)evalE(min,null)).get();
+ final long to = ((MR_long)evalE(max,null)).get();
+ return flink_env.generateSequence(from,to).map(new generator_mapper())
+ .map(new restore_global_functions());
+ case BinarySource(`file,_):
+ final String path = absolute_path(((MR_string)evalE(file,null)).get());
+ new BinaryDataSource(path,Plan.conf);
+ final FileInputFormat<FData> sf = new FlinkBinaryInputFormat().inputFormat(path);
+ return flink_env.createInput(sf).map(new restore_global_functions());
+ case ParsedSource(`parser,`file,...args):
+ final String path = absolute_path(((MR_string)evalE(file,null)).get());
+ final Class<? extends Parser> parser_class = DataSource.parserDirectory.get(parser.toString());
+ if (parser_class == null)
+ throw new Error("Unknown parser: "+parser);
+ final DataSource ds = new FlinkParsedDataSource(path,parser_class,args);
+ return flink_env.readFile(new FlinkParsedInputFormat.ParsedInputFormat(path),path)
+ .map(new restore_global_functions());
+ case Repeat(lambda(`v,`b),`s,`n):
+ final int max_num = ((MR_int)evalE(n,null)).get();
+ final IterativeDataSet<FData> startOfIteration = eval(s,env).iterate(max_num);
+ final DataSet<FData> step = eval(b,new Environment(v.toString(),new MR_flink(startOfIteration),env));
+ final DataSet<FData> toBeFedBack = step.map(new repeat_values());
+ final DataSet<FData> terminationCriterion = step.filter(new repeat_filter());
+ return startOfIteration.closeWith(toBeFedBack,terminationCriterion);
+ case Closure(lambda(`v,`b),`s,`n):
+ final int max_num = ((MR_int)evalE(n,null)).get();
+ IterativeDataSet<FData> startOfIteration = eval(s,env).iterate(max_num);
+ DataSet<FData> toBeFedBack = eval(b,new Environment(v.toString(),new MR_flink(startOfIteration),env));
+ return startOfIteration.closeWith(toBeFedBack);
+ case let(`v,`u,`body):
+ return eval(body,new Environment(v.toString(),evalE(u,null),env));
+ case Let(`v,`u,`body):
+ return eval(body,new Environment(v.toString(),new MR_flink(eval(u,env)),env));
+ case If(`c,`x,`y):
+ if (((MR_bool)evalE(c,env)).get())
+ return eval(x,env);
+ else return eval(y,env);
+ case `v:
+ if (!v.is_variable())
+ fail;
+ MRData x = variable_lookup(v.toString(),env);
+ if (x != null)
+ if (x instanceof MR_flink)
+ return ((MR_flink)x).flink();
+ else new Error("Variable "+v+" is of type: "+x);
+ x = variable_lookup(v.toString(),global_env);
+ if (x != null)
+ if (x instanceof MR_flink)
+ return ((MR_flink)x).flink();
+ else new Error("Variable "+v+" is of type: "+x);
+ throw new Error("Variable "+v+" is not bound");
+ };
+ throw new Error("Cannot evaluate the Flink plan: "+e);
+ } catch (Error msg) {
+ if (!Config.trace)
+ throw new Error(msg.getMessage());
+ System.err.println(msg.getMessage());
+ msg.printStackTrace();
+ throw new Error("Evaluation error in: "+print_query(e));
+ } catch (Exception ex) {
+ System.err.println(ex.getMessage());
+ ex.printStackTrace();
+ throw new Error("Evaluation error in: "+print_query(e));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/ea690387/flink/src/main/java/org/apache/mrql/FlinkGeneratorInputFormat.java
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkGeneratorInputFormat.java b/flink/src/main/java/org/apache/mrql/FlinkGeneratorInputFormat.java
new file mode 100644
index 0000000..68969da
--- /dev/null
+++ b/flink/src/main/java/org/apache/mrql/FlinkGeneratorInputFormat.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mrql;
+
+import org.apache.flink.api.common.io.FileInputFormat;
+
+
+/** the FileInputFormat for data generators: it creates HDFS files, where each file contains
+ * an (offset,size) pair that generates the range of values [offset,offset+size] */
+final public class FlinkGeneratorInputFormat extends FlinkMRQLFileInputFormat {
+ /** the Flink input format for this input */
+ public FileInputFormat<FData> inputFormat ( String path ) { return null; }
+}