You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by cl...@apache.org on 2013/10/15 23:33:44 UTC
git commit: updated refs/heads/trunk to b3eada3
Updated Branches:
refs/heads/trunk 768b7917f -> b3eada3c0
GIRAPH-773
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/b3eada3c
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/b3eada3c
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/b3eada3c
Branch: refs/heads/trunk
Commit: b3eada3c06deb0ca7137cad6ba8285be9d9df52c
Parents: 768b791
Author: Claudio Martella <cl...@apache.org>
Authored: Tue Oct 15 23:33:11 2013 +0200
Committer: Claudio Martella <cl...@apache.org>
Committed: Tue Oct 15 23:33:11 2013 +0200
----------------------------------------------------------------------
CHANGELOG | 6 ++++--
.../org/apache/giraph/bsp/BspOutputFormat.java | 19 +++++++++++++----
.../org/apache/giraph/conf/GiraphConstants.java | 2 +-
.../job/GiraphConfigurationValidator.java | 22 ++++++++++++++++++--
4 files changed, 40 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/b3eada3c/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 3df3164..f457026 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
Giraph Change Log
Release 1.1.0 - unreleased
+ GIRAPH-773: Cannot use only EdgeOutputFormat (armax00 via claudio)
+
GIRAPH-779: Make it easier to reuse objects with hive-io input (majakabiljo)
GIRAPH-778: Testing with TestGraph is broken (majakabiljo)
@@ -27,9 +29,9 @@ Release 1.1.0 - unreleased
GIRAPH-746: Track and log versions of dependencies (nitay)
GIRAPH-751: Build error: convertEdgeToLine in two different classes have the
- same erasure, yet neither overrides the other (aarmax0 via claudio)
+ same erasure, yet neither overrides the other (armax00 via claudio)
- GIRAPH-732: EdgeOutputFormat (aarmax00 via claudio)
+ GIRAPH-732: EdgeOutputFormat (armax00 via claudio)
GIRAPH-512: JavaDoc warnings (tdn120 via nitay)
http://git-wip-us.apache.org/repos/asf/giraph/blob/b3eada3c/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java b/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
index 7e7c65f..cdb82b2 100644
--- a/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/bsp/BspOutputFormat.java
@@ -42,12 +42,18 @@ public class BspOutputFormat extends OutputFormat<Text, Text> {
throws IOException, InterruptedException {
ImmutableClassesGiraphConfiguration conf =
new ImmutableClassesGiraphConfiguration(context.getConfiguration());
- if (!conf.hasVertexOutputFormat()) {
+ if (!conf.hasVertexOutputFormat() && !conf.hasEdgeOutputFormat()) {
LOG.warn("checkOutputSpecs: ImmutableOutputCommiter" +
" will not check anything");
return;
}
- conf.createWrappedVertexOutputFormat().checkOutputSpecs(context);
+
+ if (conf.hasVertexOutputFormat()) {
+ conf.createWrappedVertexOutputFormat().checkOutputSpecs(context);
+ }
+ if (conf.hasEdgeOutputFormat()) {
+ conf.createWrappedEdgeOutputFormat().checkOutputSpecs(context);
+ }
}
@Override
@@ -55,12 +61,17 @@ public class BspOutputFormat extends OutputFormat<Text, Text> {
throws IOException, InterruptedException {
ImmutableClassesGiraphConfiguration conf =
new ImmutableClassesGiraphConfiguration(context.getConfiguration());
- if (!conf.hasVertexOutputFormat()) {
+ if (!conf.hasVertexOutputFormat() && !conf.hasEdgeOutputFormat()) {
LOG.warn("getOutputCommitter: Returning " +
"ImmutableOutputCommiter (does nothing).");
return new ImmutableOutputCommitter();
}
- return conf.createWrappedVertexOutputFormat().getOutputCommitter(context);
+
+ if (conf.hasVertexOutputFormat()) {
+ return conf.createWrappedVertexOutputFormat().getOutputCommitter(context);
+ } else {
+ return conf.createWrappedEdgeOutputFormat().getOutputCommitter(context);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/b3eada3c/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
index 89fce61..eb8eb21 100644
--- a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
+++ b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
@@ -229,7 +229,7 @@ public interface GiraphConstants {
EdgeOutputFormat.class, "EdgeOutputFormat class");
/** EdgeOutputFormat sub-directory */
StrConfOption EDGE_OUTPUT_FORMAT_SUBDIR =
- new StrConfOption("giraph.edge.output.subdir", "edges",
+ new StrConfOption("giraph.edge.output.subdir", "",
"EdgeOutputFormat sub-directory");
/** GiraphTextOuputFormat Separator */
http://git-wip-us.apache.org/repos/asf/giraph/blob/b3eada3c/giraph-core/src/main/java/org/apache/giraph/job/GiraphConfigurationValidator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/job/GiraphConfigurationValidator.java b/giraph-core/src/main/java/org/apache/giraph/job/GiraphConfigurationValidator.java
index fcb5b87..73a7aab 100644
--- a/giraph-core/src/main/java/org/apache/giraph/job/GiraphConfigurationValidator.java
+++ b/giraph-core/src/main/java/org/apache/giraph/job/GiraphConfigurationValidator.java
@@ -28,6 +28,7 @@ import org.apache.giraph.graph.DefaultVertexResolver;
import org.apache.giraph.graph.VertexValueCombiner;
import org.apache.giraph.graph.VertexResolver;
import org.apache.giraph.io.EdgeInputFormat;
+import org.apache.giraph.io.EdgeOutputFormat;
import org.apache.giraph.io.VertexInputFormat;
import org.apache.giraph.io.VertexOutputFormat;
import org.apache.hadoop.conf.Configuration;
@@ -140,6 +141,7 @@ public class GiraphConfigurationValidator<I extends WritableComparable,
verifyVertexInputFormatGenericTypes();
verifyEdgeInputFormatGenericTypes();
verifyVertexOutputFormatGenericTypes();
+ verifyEdgeOutputFormatGenericTypes();
verifyVertexResolverGenericTypes();
verifyVertexValueCombinerGenericTypes();
verifyMessageCombinerGenericTypes();
@@ -276,11 +278,27 @@ public class GiraphConfigurationValidator<I extends WritableComparable,
}
}
- /** Verify that the output format's generic params match the job. */
+ /** Verify that the vertex output format's generic params match the job. */
private void verifyVertexOutputFormatGenericTypes() {
+ Class<? extends EdgeOutputFormat<I, V, E>>
+ edgeOutputFormatClass = conf.getEdgeOutputFormatClass();
+ if (conf.hasEdgeOutputFormat()) {
+ Class<?>[] classList =
+ getTypeArguments(EdgeOutputFormat.class, edgeOutputFormatClass);
+ checkAssignable(classList, ID_PARAM_INDEX, vertexIndexType(),
+ VertexOutputFormat.class, "vertex index");
+ checkAssignable(classList, VALUE_PARAM_INDEX, vertexValueType(),
+ VertexOutputFormat.class, "vertex value");
+ checkAssignable(classList, EDGE_PARAM_INDEX, edgeValueType(),
+ VertexOutputFormat.class, "edge value");
+ }
+ }
+
+ /** Verify that the edge output format's generic params match the job. */
+ private void verifyEdgeOutputFormatGenericTypes() {
Class<? extends VertexOutputFormat<I, V, E>>
vertexOutputFormatClass = conf.getVertexOutputFormatClass();
- if (vertexOutputFormatClass != null) {
+ if (conf.hasVertexOutputFormat()) {
Class<?>[] classList =
getTypeArguments(VertexOutputFormat.class, vertexOutputFormatClass);
checkAssignable(classList, ID_PARAM_INDEX, vertexIndexType(),