You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrql.apache.org by fe...@apache.org on 2016/02/15 16:02:29 UTC
incubator-mrql git commit: [MRQL-83] Make Hadoop 2.* (Yarn) the
default and update pom files
Repository: incubator-mrql
Updated Branches:
refs/heads/master 6e00136ca -> 807df966e
[MRQL-83] Make Hadoop 2.* (Yarn) the default and update pom files
Project: http://git-wip-us.apache.org/repos/asf/incubator-mrql/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mrql/commit/807df966
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mrql/tree/807df966
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mrql/diff/807df966
Branch: refs/heads/master
Commit: 807df966eb6b006d41a87e07f0d8609170034cd9
Parents: 6e00136
Author: fegaras <fe...@cse.uta.edu>
Authored: Sat Feb 13 15:15:32 2016 -0600
Committer: fegaras <fe...@cse.uta.edu>
Committed: Sat Feb 13 15:15:32 2016 -0600
----------------------------------------------------------------------
bsp/pom.xml | 1 +
.../main/java/org/apache/mrql/BSPEvaluator.gen | 4 ++--
conf/mrql-env.sh | 22 ++++++++++----------
flink/pom.xml | 1 +
.../java/org/apache/mrql/FlinkEvaluator.gen | 4 ++--
mapreduce/pom.xml | 1 +
.../java/org/apache/mrql/MapReduceEvaluator.gen | 6 ++----
pom.xml | 18 ++++++++++------
spark/pom.xml | 7 +++++++
9 files changed, 39 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/807df966/bsp/pom.xml
----------------------------------------------------------------------
diff --git a/bsp/pom.xml b/bsp/pom.xml
index a9b940a..3a3073d 100644
--- a/bsp/pom.xml
+++ b/bsp/pom.xml
@@ -58,6 +58,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <version>4.11</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/807df966/bsp/src/main/java/org/apache/mrql/BSPEvaluator.gen
----------------------------------------------------------------------
diff --git a/bsp/src/main/java/org/apache/mrql/BSPEvaluator.gen b/bsp/src/main/java/org/apache/mrql/BSPEvaluator.gen
index 6f6a4bf..37da613 100644
--- a/bsp/src/main/java/org/apache/mrql/BSPEvaluator.gen
+++ b/bsp/src/main/java/org/apache/mrql/BSPEvaluator.gen
@@ -36,12 +36,12 @@ final public class BSPEvaluator extends Evaluator {
conf.set("bsp.master.address","local");
conf.set("hama.zookeeper.quorum","localhost");
conf.setInt("bsp.local.tasks.maximum",Config.nodes);
- conf.set("fs.default.name","file:///");
+ conf.set("fs.defaultFS","file:///");
} else {
conf.set("bsp.master.address",System.getenv("BSP_MASTER_ADDRESS"));
conf.set("hama.zookeeper.quorum",System.getenv("HAMA_ZOOKEEPER_QUORUM"));
conf.setInt("bsp.local.tasks.maximum",Config.nodes);
- conf.set("fs.default.name",System.getenv("FS_DEFAULT_NAME"));
+ conf.set("fs.defaultFS",System.getenv("FS_DEFAULT_NAME"));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/807df966/conf/mrql-env.sh
----------------------------------------------------------------------
diff --git a/conf/mrql-env.sh b/conf/mrql-env.sh
index fac62b1..135d96c 100644
--- a/conf/mrql-env.sh
+++ b/conf/mrql-env.sh
@@ -22,11 +22,11 @@
#
# To rebuild Apache MRQL from sources:
#
-# build MRQL on Hadoop 1.x:
-# mvn -Dhadoop.version=1.0.3 clean install
-#
# build MRQL on Hadoop 2.x (yarn):
-# mvn -Dyarn -Dyarn.version=2.2.0 -Dhadoop.version=1.2.1 clean install
+# mvn -Dyarn.version=2.2.0 clean install
+#
+# build MRQL on Hadoop 1.x:
+# mvn -Dhadoop1 -Dhadoop.version=1.0.3 clean install
#
#--------------------------------------------------------------------------------
#
@@ -48,7 +48,7 @@ JLINE_JAR=${HOME}/.m2/repository/jline/jline/1.0/jline-1.0.jar
# Required: Hadoop configuration. Supports versions 1.x and 2.x (YARN)
-HADOOP_VERSION=2.6.0
+HADOOP_VERSION=2.7.1
# The Hadoop installation directory
HADOOP_HOME=${HOME}/hadoop-${HADOOP_VERSION}
# The Hadoop configuration directory (where core-site.xml is)
@@ -59,8 +59,8 @@ MAPRED_JOB_TRACKER=localhost:9001
FS_DEFAULT_NAME=hdfs://localhost:9000/
-# Optional: Hama configuration. Supports versions 0.6.2, 0.6.3, and 0.6.4
-HAMA_VERSION=0.6.4
+# Optional: Hama configuration. Supports versions 0.6.2, 0.6.3, 0.6.4, and 0.7.0
+HAMA_VERSION=0.7.0
# The Hama installation directory
HAMA_HOME=${HOME}/hama-${HAMA_VERSION}
# The Hama configuration directory
@@ -72,12 +72,12 @@ HAMA_ZOOKEEPER_QUORUM=localhost
BSP_SPLIT_INPUT=
-# Optional: Spark configuration. Supports versions 1.0.0, 1.0.2, 1.1.0, 1.1.1, 1.2.0, 1.3.0, and 1.3.1
+# Optional: Spark configuration. Supports versions 1.0.0, 1.0.2, 1.1.0, 1.1.1, 1.2.0, 1.3.0, 1.3.1, and 1.6.0
# (Spark versions 0.8.1, 0.9.0, and 0.9.1 are supported by MRQL 0.9.0)
# You may use the Spark prebuilts bin-hadoop1 or bin-hadoop2 (Yarn)
# For distributed mode, give write permission to /tmp: hadoop fs -chmod -R 777 /tmp
# Tested in local, standalone deploy, and Yarn modes
-SPARK_HOME=${HOME}/spark-1.3.0-bin-hadoop2.4
+SPARK_HOME=${HOME}/spark-1.6.0-bin-hadoop2.6
# URI of the Spark master node:
# to run Spark on Standalone Mode, set it to spark://`hostname`:7077
# to run Spark on a YARN cluster, set it to "yarn-client"
@@ -93,8 +93,8 @@ SPARK_EXECUTOR_CORES=1
SPARK_EXECUTOR_MEMORY=1G
-# Optional: Flink configuration. Supports version 0.10.1
-FLINK_VERSION=0.10.1
+# Optional: Flink configuration. Supports versions 0.10.1 and 0.10.2
+FLINK_VERSION=0.10.2
# Flink installation directory
FLINK_HOME=${HOME}/flink-${FLINK_VERSION}
# number of slots per TaskManager (typically, the number of CPUs per machine)
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/807df966/flink/pom.xml
----------------------------------------------------------------------
diff --git a/flink/pom.xml b/flink/pom.xml
index 3b01456..9243682 100644
--- a/flink/pom.xml
+++ b/flink/pom.xml
@@ -68,6 +68,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <version>4.11</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/807df966/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen b/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
index 83e80c2..1f2637f 100644
--- a/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
+++ b/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
@@ -68,7 +68,7 @@ public class FlinkEvaluator extends Evaluator implements Serializable {
// curently, the compiler doesn't work in local mode
Config.compile_functional_arguments = false;
fs_default_name = "file:///";
- Plan.conf.set("fs.default.name",fs_default_name);
+ Plan.conf.set("fs.defaultFS",fs_default_name);
} else if (Config.distributed_mode) {
String master_node = System.getenv("FLINK_MASTER");
if (master_node == null)
@@ -82,7 +82,7 @@ public class FlinkEvaluator extends Evaluator implements Serializable {
};
fs_default_name = System.getenv("FS_DEFAULT_NAME");
data_source_dir_name = absolute_path("tmp/data_source_dir.txt");
- Plan.conf.set("fs.default.name",fs_default_name);
+ Plan.conf.set("fs.defaultFS",fs_default_name);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/807df966/mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/mapreduce/pom.xml b/mapreduce/pom.xml
index a54799c..448bf67 100644
--- a/mapreduce/pom.xml
+++ b/mapreduce/pom.xml
@@ -53,6 +53,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <version>4.11</version>
<scope>test</scope>
</dependency>
<!-- needed for testing in local mode -->
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/807df966/mapreduce/src/main/java/org/apache/mrql/MapReduceEvaluator.gen
----------------------------------------------------------------------
diff --git a/mapreduce/src/main/java/org/apache/mrql/MapReduceEvaluator.gen b/mapreduce/src/main/java/org/apache/mrql/MapReduceEvaluator.gen
index 1fdc5cb..ac6e93a 100644
--- a/mapreduce/src/main/java/org/apache/mrql/MapReduceEvaluator.gen
+++ b/mapreduce/src/main/java/org/apache/mrql/MapReduceEvaluator.gen
@@ -35,12 +35,10 @@ final public class MapReduceEvaluator extends Evaluator {
if (Config.local_mode) {
conf.set("mapred.job.tracker","local");
conf.set("mapreduce.framework.name","local");
- conf.set("fs.default.name","file:///");
- //conf.set("fs.defaultFS","file:///"); // doesn't work for hadoop 1.*
+ conf.set("fs.defaultFS","file:///"); // doesn't work for hadoop 1.*
} else {
conf.set("mapred.job.tracker",System.getenv("MAPRED_JOB_TRACKER"));
- conf.set("fs.default.name",System.getenv("FS_DEFAULT_NAME"));
- //conf.set("fs.defaultFS",System.getenv("FS_DEFAULT_NAME")); // doesn't work for hadoop 1.*
+ conf.set("fs.defaultFS",System.getenv("FS_DEFAULT_NAME")); // doesn't work for hadoop 1.*
}
}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/807df966/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f1f0029..568e891 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,11 +45,11 @@
<properties>
<hadoop.version>1.2.1</hadoop.version>
- <yarn.version>2.6.0</yarn.version>
- <hama.version>0.6.4</hama.version>
- <spark.version>1.3.1</spark.version>
+ <yarn.version>2.7.1</yarn.version>
+ <hama.version>0.7.0</hama.version>
+ <spark.version>1.6.0</spark.version>
<scala.version>2.10</scala.version>
- <flink.version>0.10.1</flink.version>
+ <flink.version>0.10.2</flink.version>
<skipTests>true</skipTests>
</properties>
@@ -79,6 +79,12 @@
<artifactId>jline</artifactId>
<version>1.0</version>
</dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<dependencyManagement>
@@ -210,7 +216,7 @@
<id>hadoop1</id>
<activation>
<property>
- <name>!yarn</name>
+ <name>hadoop1</name>
</property>
</activation>
<dependencies>
@@ -230,7 +236,7 @@
<id>yarn</id>
<activation>
<property>
- <name>yarn</name>
+ <name>!hadoop1</name>
</property>
</activation>
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/807df966/spark/pom.xml
----------------------------------------------------------------------
diff --git a/spark/pom.xml b/spark/pom.xml
index 4c7ce91..e36b49e 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -63,8 +63,15 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <version>4.11</version>
<scope>test</scope>
</dependency>
+ <!-- needed for testing in local mode -->
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ </dependency>
</dependencies>
<build>