You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrql.apache.org by fe...@apache.org on 2014/06/12 18:15:25 UTC
git commit: MRQL-38: Support for Spark 1.0.0
Repository: incubator-mrql
Updated Branches:
refs/heads/master 85adb0bd4 -> 8305578c2
MRQL-38: Support for Spark 1.0.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-mrql/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mrql/commit/8305578c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mrql/tree/8305578c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mrql/diff/8305578c
Branch: refs/heads/master
Commit: 8305578c242e961f46a3ed6fa4e41c7f0757d372
Parents: 85adb0b
Author: fegaras <fe...@cse.uta.edu>
Authored: Thu Jun 12 19:13:36 2014 +0300
Committer: fegaras <fe...@cse.uta.edu>
Committed: Thu Jun 12 19:13:36 2014 +0300
----------------------------------------------------------------------
README | 2 +-
RELEASE_NOTES | 33 ++++++++++++++
bsp/pom.xml | 2 +-
conf/mrql-env.sh | 16 +++++--
core/pom.xml | 2 +-
dist/pom.xml | 2 +-
gen/pom.xml | 2 +-
mapreduce/pom.xml | 2 +-
pom.xml | 6 +--
spark/pom.xml | 2 +-
.../java/org/apache/mrql/SparkEvaluator.gen | 48 ++++++++++----------
11 files changed, 77 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/README
----------------------------------------------------------------------
diff --git a/README b/README
index 766764b..231dbd2 100644
--- a/README
+++ b/README
@@ -18,7 +18,7 @@
***************************************************************************
-Apache MRQL 0.9.0
+Apache MRQL 0.9.2
=================
Apache MRQL (pronounced miracle) is a query processing and optimization
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/RELEASE_NOTES
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 56eeefa..a6addf1 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -16,7 +16,40 @@
limitations under the License.
***************************************************************************
+Release Notes - Apache MRQL 0.9.2
+---------------------------------
+
+New Features
+
+[MRQL-23] Add support for Yarn
+[MRQL-29] Support for Spark 0.9.0
+
+Bugs
+
+[MRQL-22] The compiler must load the hadoop core lib before compilation
+[MRQL-28] Fixed dumping of HDFS sequence file in Spark mode
+[MRQL-31] Broken hyperlink in website
+[MRQL-33] Fix various bugs in iteration queries
+[MRQL-36] Fix evaluation errors during Junit testing
+
+Improvements
+
+[MRQL-21] Modify shell scripts to work on Mac OS
+[MRQL-24] Improve Serializable methods for Spark
+[MRQL-25] Changed Translator/Evaluator interface to improve Spark efficiency
+[MRQL-26] Support for Spark 0.8.1
+[MRQL-27] Improve the build process and support hadoop 0.20.x
+[MRQL-30] Improved run scripts and support for Spark on YARN
+[MRQL-32] Refactoring directory structure for Eclipse
+[MRQL-34] Introduce junit for testing
+[MRQL-37] Support installation on a Cloudera CDH distribution
+
+Tasks
+
+[MRQL-20] Fix maven problems to prepare a new release candidate
+
Release Notes - Apache MRQL 0.9.0
+---------------------------------
New Features
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/bsp/pom.xml
----------------------------------------------------------------------
diff --git a/bsp/pom.xml b/bsp/pom.xml
index fee0a92..910c5dc 100644
--- a/bsp/pom.xml
+++ b/bsp/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.mrql</groupId>
<artifactId>mrql-parent</artifactId>
- <version>0.9.1-incubating-SNAPSHOT</version>
+ <version>0.9.2-incubating-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/conf/mrql-env.sh
----------------------------------------------------------------------
diff --git a/conf/mrql-env.sh b/conf/mrql-env.sh
index 2451105..56274d6 100644
--- a/conf/mrql-env.sh
+++ b/conf/mrql-env.sh
@@ -49,7 +49,7 @@ JLINE_JAR=${HOME}/.m2/repository/jline/jline/1.0/jline-1.0.jar
# Required: Hadoop configuration. Supports versions 0.20.x, 1.x, and 2.x (YARN)
-HADOOP_VERSION=1.2.1
+HADOOP_VERSION=2.2.0
# The Hadoop installation directory
HADOOP_HOME=${HOME}/hadoop-${HADOOP_VERSION}
# The Hadoop job tracker (as defined in mapred-site.xml)
@@ -58,7 +58,7 @@ MAPRED_JOB_TRACKER=localhost:9001
FS_DEFAULT_NAME=hdfs://localhost:9000/
-# Optional: Hama configuration. Supports versions 0.5.0, 0.6.0, 0.6.2, and 0.6.3
+# Optional: Hama configuration. Supports versions 0.6.2 and 0.6.3 (but not 0.6.4)
HAMA_VERSION=0.6.3
# The Hadoop installation directory
HAMA_HOME=${HOME}/hama-${HAMA_VERSION}
@@ -67,10 +67,11 @@ BSP_MASTER_ADDRESS=localhost:40000
HAMA_ZOOKEEPER_QUORUM=localhost
-# Optional: Spark configuration. Supports versions 0.8.1 and 0.9.0
+# Optional: Spark configuration. Supports version 1.0.0 only
+# (Spark versions 0.8.1, 0.9.0, and 0.9.1 are supported by MRQL 0.9.0)
# Use either the Spark prebuilts bin-hadoop1 or bin-hadoop2 (Yarn)
# Tested in local, standalone deploy, and Yarn modes
-SPARK_HOME=${HOME}/spark-0.9.0-incubating-bin-hadoop1
+SPARK_HOME=${HOME}/spark-1.0.0-bin-hadoop2
# URI of the Spark master node (to run Spark on a YARN cluster, set it to "yarn-client")
SPARK_MASTER=spark://crete:7077
# For a Yarn cluster set it to the number of workers to start on,
@@ -89,7 +90,12 @@ HAMA_JAR=${HAMA_HOME}/hama-core-${HAMA_VERSION}.jar
# YARN-enabled assembly jar
if [[ -d ${SPARK_HOME}/assembly/target ]]; then
- SPARK_JAR=`ls ${SPARK_HOME}/assembly/target/scala-*/*`
+ # old Spark (0.x)
+ SPARK_JAR=`ls ${SPARK_HOME}/assembly/target/scala-*/*.jar`
+else if [[ -d ${SPARK_HOME}/lib ]]; then
+ # new Spark (1.x)
+ SPARK_JAR=`ls ${SPARK_HOME}/lib/spark-assembly-*.jar`
+fi
fi
if [[ -f ${HADOOP_HOME}/share/hadoop/common/hadoop-common-${HADOOP_VERSION}.jar ]]; then
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 3c55f0b..6dc7a3d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.mrql</groupId>
<artifactId>mrql-parent</artifactId>
- <version>0.9.1-incubating-SNAPSHOT</version>
+ <version>0.9.2-incubating-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/dist/pom.xml
----------------------------------------------------------------------
diff --git a/dist/pom.xml b/dist/pom.xml
index 21b95ba..c9d49d1 100644
--- a/dist/pom.xml
+++ b/dist/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.mrql</groupId>
<artifactId>mrql-parent</artifactId>
- <version>0.9.1-incubating-SNAPSHOT</version>
+ <version>0.9.2-incubating-SNAPSHOT</version>
</parent>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/gen/pom.xml
----------------------------------------------------------------------
diff --git a/gen/pom.xml b/gen/pom.xml
index d49ad1c..799c897 100644
--- a/gen/pom.xml
+++ b/gen/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.mrql</groupId>
<artifactId>mrql-parent</artifactId>
- <version>0.9.1-incubating-SNAPSHOT</version>
+ <version>0.9.2-incubating-SNAPSHOT</version>
</parent>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/mapreduce/pom.xml b/mapreduce/pom.xml
index 5b2c9c0..2622b35 100644
--- a/mapreduce/pom.xml
+++ b/mapreduce/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.mrql</groupId>
<artifactId>mrql-parent</artifactId>
- <version>0.9.1-incubating-SNAPSHOT</version>
+ <version>0.9.2-incubating-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4d743b5..c669689 100644
--- a/pom.xml
+++ b/pom.xml
@@ -20,7 +20,7 @@
<groupId>org.apache.mrql</groupId>
<artifactId>mrql-parent</artifactId>
- <version>0.9.1-incubating-SNAPSHOT</version>
+ <version>0.9.2-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache MRQL parent POM</name>
<description>Apache MRQL is a query processing and optimization system for large-scale, distributed data analysis, built on top of Apache Hadoop, Hama, and Spark</description>
@@ -46,8 +46,8 @@
<properties>
<hadoop.version>1.2.1</hadoop.version>
<yarn.version>2.2.0</yarn.version>
- <hama.version>0.6.4</hama.version>
- <spark.version>0.9.1</spark.version>
+ <hama.version>0.6.3</hama.version>
+ <spark.version>1.0.0</spark.version>
<scala.version>2.10</scala.version>
<skipTests>true</skipTests>
</properties>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/spark/pom.xml
----------------------------------------------------------------------
diff --git a/spark/pom.xml b/spark/pom.xml
index 8284e5e..2196a58 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.mrql</groupId>
<artifactId>mrql-parent</artifactId>
- <version>0.9.1-incubating-SNAPSHOT</version>
+ <version>0.9.2-incubating-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/8305578c/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen b/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
index 1bf814b..1d0f386 100644
--- a/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
+++ b/spark/src/main/java/org/apache/mrql/SparkEvaluator.gen
@@ -177,13 +177,12 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
ftp.close();
// in Spark mode, a dataset has always one RDD datasource
JavaRDD<MRData> rdd = ((RDDDataSource)((MR_dataset)data).dataset().source.get(0)).rdd.cache();
- rdd.map(new PairFunction<MRData,MRContainer,MRContainer>() {
- final MRContainer zero = new MRContainer(new MR_int(0));
- public Tuple2<MRContainer,MRContainer> call ( MRData value ) {
- return new Tuple2<MRContainer,MRContainer>(zero,new MRContainer(value));
- }
- })
- .saveAsHadoopFile(file,MRContainer.class,MRContainer.class,SequenceFileOutputFormat.class);
+ rdd.mapToPair(new PairFunction<MRData,MRContainer,MRContainer>() {
+ final MRContainer zero = new MRContainer(new MR_int(0));
+ public Tuple2<MRContainer,MRContainer> call ( MRData value ) {
+ return new Tuple2<MRContainer,MRContainer>(zero,new MRContainer(value));
+ }
+ }).saveAsHadoopFile(file,MRContainer.class,MRContainer.class,SequenceFileOutputFormat.class);
} else super.dump(file,type,data);
}
@@ -287,7 +286,7 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
throw new Error("Wrong Loop format");
}
- private static Bag bag ( final List<MRData> s ) {
+ private static Bag bag ( final Iterable<MRData> s ) {
final Iterator<MRData> i = s.iterator();
return new Bag(new BagIterator() {
public MRData next () {
@@ -300,7 +299,7 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
}
// changed since Spark 0.8.1
- final static TaskContext context = new TaskContext(0,0,(long)0,Config.local_mode,false,null);
+ final static TaskContext context = new TaskContext(0,0,(long)0,Config.local_mode,null);
/** Convert a Spark RDD into a lazy bag
* @param rdd the Spark RDD
@@ -398,11 +397,10 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
case true: // the result must be sorted
return s.groupBy(get_first)
.sortByKey()
- .map(new Function<Tuple2<MRData,List<MRData>>,MRData>() {
- public MRData call ( Tuple2<MRData,List<MRData>> value ) {
+ .map(new Function<Tuple2<MRData,Iterable<MRData>>,MRData>() {
+ public MRData call ( Tuple2<MRData,Iterable<MRData>> value ) {
return new Tuple(value._1,bag(value._2));
- }
- })
+ }})
.flatMap(cmap_fnc(fnc,env)).map(new Function<MRData,MRData>() {
public MRData call ( MRData value ) {
return ((Tuple)value).second();
@@ -410,8 +408,8 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
});
};
return s.groupBy(get_first)
- .map(new Function<Tuple2<MRData,List<MRData>>,MRData> () {
- public MRData call ( Tuple2<MRData,List<MRData>> value ) {
+ .map(new Function<Tuple2<MRData,Iterable<MRData>>,MRData> () {
+ public MRData call ( Tuple2<MRData,Iterable<MRData>> value ) {
final Iterator<MRData> i = value._2.iterator();
return new Tuple(value._1,new Bag(new BagIterator() {
public MRData next () {
@@ -538,20 +536,20 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
final org.apache.mrql.Function fy = evalF(my,env);
final org.apache.mrql.Function fr = evalF(r,env);
JavaPairRDD<MRData,MRData> xs
- = eval(x,env).flatMap(new PairFlatMapFunction<MRData,MRData,MRData>() {
+ = eval(x,env).flatMapToPair(new PairFlatMapFunction<MRData,MRData,MRData>() {
public Iterable<Tuple2<MRData,MRData>> call ( MRData value ) {
return joinIterator(((Bag)fx.eval(value)).iterator());
}
});
JavaPairRDD<MRData,MRData> ys
- = eval(y,env).flatMap(new PairFlatMapFunction<MRData,MRData,MRData>() {
+ = eval(y,env).flatMapToPair(new PairFlatMapFunction<MRData,MRData,MRData>() {
public Iterable<Tuple2<MRData,MRData>> call ( MRData value ) {
return joinIterator(((Bag)fy.eval(value)).iterator());
}
});
return xs.cogroup(ys)
- .flatMap(new FlatMapFunction<Tuple2<MRData,Tuple2<List<MRData>,List<MRData>>>,MRData>() {
- public Iterable<MRData> call ( Tuple2<MRData,Tuple2<List<MRData>,List<MRData>>> value ) {
+ .flatMap(new FlatMapFunction<Tuple2<MRData,Tuple2<Iterable<MRData>,Iterable<MRData>>>,MRData>() {
+ public Iterable<MRData> call ( Tuple2<MRData,Tuple2<Iterable<MRData>,Iterable<MRData>>> value ) {
return (Bag)fr.eval(new Tuple(bag(value._2._1),bag(value._2._2)));
}
});
@@ -569,7 +567,7 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
final MRData one = new MR_byte(1);
final MRData two = new MR_byte(2);
final JavaPairRDD<MRData,MRData> xs
- = eval(x,env).flatMap(new PairFlatMapFunction<MRData,MRData,MRData>() {
+ = eval(x,env).flatMapToPair(new PairFlatMapFunction<MRData,MRData,MRData>() {
public Iterable<Tuple2<MRData,MRData>> call ( final MRData value ) {
return new Iterable<Tuple2<MRData,MRData>>() {
public Iterator<Tuple2<MRData,MRData>> iterator() {
@@ -590,7 +588,7 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
}
});
final JavaPairRDD<MRData,MRData> ys
- = eval(y,env).flatMap(new PairFlatMapFunction<MRData,MRData,MRData>() {
+ = eval(y,env).flatMapToPair(new PairFlatMapFunction<MRData,MRData,MRData>() {
public Iterable<Tuple2<MRData,MRData>> call ( final MRData value ) {
return new Iterable<Tuple2<MRData,MRData>>() {
public Iterator<Tuple2<MRData,MRData>> iterator() {
@@ -611,12 +609,12 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
}
});
return xs.union(ys).groupByKey(Config.nodes)
- .mapPartitions(new FlatMapFunction<Iterator<Tuple2<MRData,List<MRData>>>,MRData>() {
- public Iterable<MRData> call ( final Iterator<Tuple2<MRData,List<MRData>>> value ) {
+ .mapPartitions(new FlatMapFunction<Iterator<Tuple2<MRData,Iterable<MRData>>>,MRData>() {
+ public Iterable<MRData> call ( final Iterator<Tuple2<MRData,Iterable<MRData>>> value ) {
Bag xb = new Bag();
Bag yb = new Bag();
while (value.hasNext()) {
- Tuple2<MRData,List<MRData>> t = value.next();
+ Tuple2<MRData,Iterable<MRData>> t = value.next();
for ( MRData e: t._2 ) {
Tuple p = (Tuple)e;
if (((MR_byte)p.first()).get() == 1)
@@ -638,7 +636,7 @@ final public class SparkEvaluator extends Evaluator implements Serializable {
final org.apache.mrql.Function fy = evalF(my,env);
final org.apache.mrql.Function fr = evalF(r,env);
final Broadcast<List<Tuple2<MRData,MRData>>> ys
- = spark_context.broadcast(eval(y,env).flatMap(new PairFlatMapFunction<MRData,MRData,MRData>() {
+ = spark_context.broadcast(eval(y,env).flatMapToPair(new PairFlatMapFunction<MRData,MRData,MRData>() {
public Iterable<Tuple2<MRData,MRData>> call ( MRData value ) {
return joinIterator(((Bag)fy.eval(value)).iterator());
}