You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2013/05/07 06:04:25 UTC
git commit: updated refs/heads/trunk to fd9690a
Updated Branches:
refs/heads/trunk a419a7bc4 -> fd9690a05
GIRAPH-655: Fix hadoop 2.0.0 build (nitay)
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/fd9690a0
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/fd9690a0
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/fd9690a0
Branch: refs/heads/trunk
Commit: fd9690a05de75707de59f5c44d56373bed500819
Parents: a419a7b
Author: Nitay Joffe <ni...@apache.org>
Authored: Mon May 6 18:30:31 2013 -0400
Committer: Nitay Joffe <ni...@apache.org>
Committed: Tue May 7 00:03:37 2013 -0400
----------------------------------------------------------------------
.gitignore | 1 +
CHANGELOG | 2 +
dev-support/for-each-profile.sh | 49 +++-----
.../java/org/apache/giraph/job/HadoopUtils.java | 87 +++++++++++++++
.../test/java/org/apache/giraph/TestBspBasic.java | 30 ++----
.../apache/giraph/hive/output/HiveOutputTest.java | 11 +-
pom.xml | 7 +-
7 files changed, 127 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 8c911c1..42ecd00 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,4 +29,5 @@ failed-profile.txt
# Vim auto-save files:
.*.swp
+/for-each-profile-results.txt
/giraph-hive/derby.log
http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 954401c..e81a478 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
Giraph Change Log
Release 1.1.0 - unreleased
+ GIRAPH-655: Fix Hive Build for Hadoop 2.0.0 (nitay)
+
GIRAPH-660: Move the snapshot to 1.1.0 and also add a releases page for
the 1.0 release (aching)
http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/dev-support/for-each-profile.sh
----------------------------------------------------------------------
diff --git a/dev-support/for-each-profile.sh b/dev-support/for-each-profile.sh
index cefafb2..4917fb9 100755
--- a/dev-support/for-each-profile.sh
+++ b/dev-support/for-each-profile.sh
@@ -49,41 +49,28 @@ parse_profiles
echo "Running on profiles: $all_profiles"
-failed_profile_file=$TOP_DIR/failed-profile.txt
-if [[ -r "$failed_profile_file" ]]; then
- echo "Reading $failed_profile_file to restart from last run state."
- echo "Remove that file if you want a fresh run"
- failed_profile=$(cat $failed_profile_file)
- run_profile=false
-else
- run_profile=true
-fi
+results_file=$TOP_DIR/for-each-profile-results.txt
+
+echo "mvn -P<profile> $@" > $results_file
+echo "======================================================" >> $results_file
for profile in $all_profiles; do
- if ! $run_profile; then
- if [[ "${profile}" == "${failed_profile}" ]]; then
- run_profile=true
- fi
- fi
- if $run_profile; then
- echo_separator
- echo "=== $profile ==="
- echo_separator
+ echo_separator
+ echo "=== $profile ==="
+ echo_separator
- mvn -P$profile $@
- result=$?
-
- if [[ $result -ne 0 ]]; then
- echo_separator
- echo "=== Failed on profile: $profile"
- echo "=== Failed command: mvn -P$profile $@"
- echo_separator
- echo $profile > $failed_profile_file
- exit $result
- fi
- else
+ args="-P$profile $@"
+
+ mvn $args
+ result=$?
+
+ if [[ $result -ne 0 ]]; then
echo_separator
- echo "=== Skipping profile $profile"
+ echo "=== Failed on profile: $profile"
+ echo "=== Failed command: mvn $args"
+ echo "$profile: FAILED" >> $results_file
echo_separator
+ else
+ echo "$profile: SUCCESS" >> $results_file
fi
done
http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/giraph-core/src/main/java/org/apache/giraph/job/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/job/HadoopUtils.java b/giraph-core/src/main/java/org/apache/giraph/job/HadoopUtils.java
new file mode 100644
index 0000000..d5095bc
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/job/HadoopUtils.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.job;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/*if_not[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+/*end[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/
+
+/**
+ * Helpers for dealing with Hadoop.
+ */
+public class HadoopUtils {
+ /** Don't construct */
+ private HadoopUtils() { }
+
+ /**
+ * Create a TaskAttemptContext, supporting many Hadoops.
+ *
+ * @param conf Configuration
+ * @return TaskAttemptContext
+ */
+ public static TaskAttemptContext makeTaskAttemptContext(Configuration conf) {
+ TaskAttemptContext context;
+ /*if[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]
+ context = new TaskAttemptContext(conf, new TaskAttemptID());
+ else[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/
+ context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
+ /*end[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/
+ return context;
+ }
+
+ /**
+ * Create a TaskAttemptContext, supporting many Hadoops.
+ *
+ * @return TaskAttemptContext
+ */
+ public static TaskAttemptContext makeTaskContext() {
+ return makeTaskAttemptContext(new Configuration());
+ }
+
+ /**
+ * Create a JobContext, supporting many Hadoops.
+ *
+ * @param conf Configuration
+ * @return JobContext
+ */
+ public static JobContext makeJobContext(Configuration conf) {
+ JobContext context;
+ /*if[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]
+ context = new JobContext(conf, new JobID());
+ else[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/
+ context = new JobContextImpl(conf, new JobID());
+ /*end[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/
+ return context;
+ }
+
+ /**
+ * Create a JobContext, supporting many Hadoops.
+ *
+ * @return JobContext
+ */
+ public static JobContext makeJobContext() {
+ return makeJobContext(new Configuration());
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
----------------------------------------------------------------------
diff --git a/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java b/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
index aa3b08c..38ba27f 100644
--- a/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
+++ b/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
@@ -18,7 +18,8 @@
package org.apache.giraph;
-import org.apache.giraph.conf.GiraphClasses;
+import org.apache.giraph.aggregators.TextAggregatorWriter;
+import org.apache.giraph.combiner.SimpleSumCombiner;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
@@ -30,23 +31,21 @@ import org.apache.giraph.examples.SimpleMsgVertex;
import org.apache.giraph.examples.SimplePageRankVertex;
import org.apache.giraph.examples.SimplePageRankVertex.SimplePageRankVertexInputFormat;
import org.apache.giraph.examples.SimpleShortestPathsVertex;
-import org.apache.giraph.combiner.SimpleSumCombiner;
import org.apache.giraph.examples.SimpleSuperstepVertex;
import org.apache.giraph.examples.SimpleSuperstepVertex.SimpleSuperstepVertexInputFormat;
import org.apache.giraph.examples.SimpleSuperstepVertex.SimpleSuperstepVertexOutputFormat;
import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.job.GiraphJob;
-import org.apache.giraph.worker.InputSplitPathOrganizer;
-import org.apache.giraph.aggregators.TextAggregatorWriter;
import org.apache.giraph.io.VertexInputFormat;
import org.apache.giraph.io.formats.JsonLongDoubleFloatDoubleVertexOutputFormat;
+import org.apache.giraph.job.GiraphJob;
+import org.apache.giraph.job.HadoopUtils;
+import org.apache.giraph.worker.InputSplitPathOrganizer;
import org.apache.giraph.zk.ZooKeeperExt;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
@@ -54,7 +53,7 @@ import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobContext;
import org.apache.zookeeper.KeeperException;
import org.junit.Test;
@@ -81,12 +80,6 @@ import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-/*if[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]
-import org.apache.hadoop.mapreduce.JobContext;
-else[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/
-import org.apache.hadoop.mapreduce.task.JobContextImpl;
-/*end[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/
-
/**
* Unit test for many simple BSP applications.
*/
@@ -130,15 +123,8 @@ public class
System.out.println("testInstantiateVertex: Got vertex " + vertex);
VertexInputFormat<LongWritable, IntWritable, FloatWritable>
inputFormat = configuration.createWrappedVertexInputFormat();
-/*if[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]
- List<InputSplit> splitArray =
- inputFormat.getSplits(
- new JobContext(new Configuration(), new JobID()), 1);
-else[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/
- List<InputSplit> splitArray =
- inputFormat.getSplits(
- new JobContextImpl(new Configuration(), new JobID()), 1);
-/*end[HADOOP_NON_JOBCONTEXT_IS_INTERFACE]*/
+ List<InputSplit> splitArray = inputFormat.getSplits(
+ HadoopUtils.makeJobContext(), 1);
ByteArrayOutputStream byteArrayOutputStream =
new ByteArrayOutputStream();
DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream);
http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java
index 5054728..673c9d5 100644
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java
+++ b/giraph-hive/src/test/java/org/apache/giraph/hive/output/HiveOutputTest.java
@@ -28,11 +28,12 @@ import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat;
import org.apache.giraph.io.internal.WrappedVertexOutputFormat;
import org.apache.giraph.utils.InternalVertexRunner;
import org.apache.hadoop.mapred.HackJobContext;
+import org.apache.hadoop.mapred.HackTaskAttemptContext;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.thrift.TException;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -124,11 +125,11 @@ public class HiveOutputTest {
throws IOException, InterruptedException {
ImmutableClassesGiraphConfiguration iconf = new ImmutableClassesGiraphConfiguration(conf);
WrappedVertexOutputFormat outputFormat = iconf.createWrappedVertexOutputFormat();
- TaskAttemptID taskID = new TaskAttemptID();
- TaskAttemptContext taskContext = new TaskAttemptContext(new JobConf(conf), taskID);
- OutputCommitter outputCommitter = outputFormat.getOutputCommitter(taskContext);
JobConf jobConf = new JobConf(conf);
- JobContext jobContext = new HackJobContext(jobConf, taskID.getJobID());
+ TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID());
+ OutputCommitter outputCommitter = outputFormat.getOutputCommitter(
+ taskContext);
+ JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID());
outputCommitter.commitJob(jobContext);
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fd9690a0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3971076..67dc913 100644
--- a/pom.xml
+++ b/pom.xml
@@ -254,7 +254,7 @@ under the License.
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <dep.hiveio.version>0.9</dep.hiveio.version>
+ <dep.hiveio.version>0.10</dep.hiveio.version>
<hbase.version>0.90.5</hbase.version>
<codehaus-jackson.version>1.8.0</codehaus-jackson.version>
<fasterxml-jackson.version>2.1.0</fasterxml-jackson.version>
@@ -596,6 +596,7 @@ under the License.
<module>giraph-accumulo</module>
<module>giraph-hbase</module>
<module>giraph-hcatalog</module>
+ <module>giraph-hive</module>
</modules>
<activation>
<activeByDefault>true</activeByDefault>
@@ -625,6 +626,7 @@ under the License.
<module>giraph-accumulo</module>
<module>giraph-hbase</module>
<module>giraph-hcatalog</module>
+ <module>giraph-hive</module>
</modules>
<properties>
<hadoop.version>1.0.2</hadoop.version>
@@ -651,6 +653,7 @@ under the License.
<module>giraph-accumulo</module>
<module>giraph-hbase</module>
<module>giraph-hcatalog</module>
+ <module>giraph-hive</module>
</modules>
<properties>
<hadoop.version>0.20.2</hadoop.version>
@@ -677,6 +680,7 @@ under the License.
<module>giraph-accumulo</module>
<module>giraph-hbase</module>
<module>giraph-hcatalog</module>
+ <module>giraph-hive</module>
</modules>
<properties>
<hadoop.version>0.20.0</hadoop.version>
@@ -1232,7 +1236,6 @@ under the License.
<modules>
<module>giraph-core</module>
- <module>giraph-hive</module>
<module>giraph-examples</module>
</modules>