You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2013/03/11 21:22:21 UTC
git commit: GIRAPH-550: HiveVertexReader should call initialize,
not users (nitay).
Updated Branches:
refs/heads/trunk 40e201df9 -> 0e52d19f1
GIRAPH-550: HiveVertexReader should call initialize, not users (nitay).
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/0e52d19f
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/0e52d19f
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/0e52d19f
Branch: refs/heads/trunk
Commit: 0e52d19f132be6078526e8ebb3a4db8f0ef40cb1
Parents: 40e201d
Author: Nitay Joffe <ni...@apache.org>
Authored: Thu Feb 28 15:32:20 2013 -0500
Committer: Nitay Joffe <ni...@apache.org>
Committed: Mon Mar 11 16:21:39 2013 -0400
----------------------------------------------------------------------
CHANGELOG | 2 +
checkstyle.xml | 1 -
giraph-accumulo/pom.xml | 30 ---
giraph-core/pom.xml | 16 --
giraph-examples/pom.xml | 30 ---
giraph-hbase/pom.xml | 30 ---
giraph-hcatalog/pom.xml | 30 ---
giraph-hive/pom.xml | 30 ---
.../org/apache/giraph/hive/HiveGiraphRunner.java | 137 +++++++--------
.../giraph/hive/input/edge/AbstractHiveToEdge.java | 38 ++++
.../giraph/hive/input/edge/HiveEdgeReader.java | 33 ++--
.../apache/giraph/hive/input/edge/HiveToEdge.java | 6 +-
.../input/vertex/AbstractHiveToVertexEdges.java | 38 ++++
.../input/vertex/AbstractHiveToVertexValue.java | 37 ++++
.../giraph/hive/input/vertex/HiveToVertex.java | 45 -----
.../hive/input/vertex/HiveToVertexEdges.java | 67 +++++++
.../hive/input/vertex/HiveToVertexValue.java | 49 +++++
.../giraph/hive/input/vertex/HiveVertexReader.java | 63 ++++++--
.../giraph/hive/output/AbstractVertexToHive.java | 38 ++++
.../giraph/hive/output/HiveVertexWriter.java | 5 +-
.../apache/giraph/hive/output/VertexToHive.java | 3 +-
pom.xml | 41 +++++-
22 files changed, 449 insertions(+), 320 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index f06627f..ee894ba 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
Giraph Change Log
Release 0.2.0 - unreleased
+ GIRAPH-550: HiveVertexReader should call initialize, not users (nitay)
+
GIRAPH-523: MasterObserver should pass in superstep number (gaurav.menghani via nitay)
GIRAPH-556: Race condition in EdgeStore (apresta)
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/checkstyle.xml
----------------------------------------------------------------------
diff --git a/checkstyle.xml b/checkstyle.xml
index 10a8b43..3d8a6d4 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -86,7 +86,6 @@
<!-- Checks for common coding problems -->
<!-- See http://checkstyle.sf.net/config_coding.html -->
- <module name="DoubleCheckedLocking"/>
<module name="EmptyStatement"/>
<!-- Require hash code override when equals is -->
<module name="EqualsHashCode"/>
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-accumulo/pom.xml b/giraph-accumulo/pom.xml
index 1f7e84a..1f229da 100644
--- a/giraph-accumulo/pom.xml
+++ b/giraph-accumulo/pom.xml
@@ -44,22 +44,6 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <configLocation>${top.dir}/checkstyle.xml</configLocation>
- <headerLocation>${top.dir}/license-header.txt</headerLocation>
- <enableRulesSummary>false</enableRulesSummary>
- <failOnError>true</failOnError>
- <includeTestSourceDirectory>false</includeTestSourceDirectory>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -81,20 +65,6 @@ under the License.
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
- <version>2.5.1</version>
- <configuration>
- <xmlOutput>true</xmlOutput>
- <findbugsXmlOutput>false</findbugsXmlOutput>
- <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-core/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-core/pom.xml b/giraph-core/pom.xml
index 2a500d8..3580d0c 100644
--- a/giraph-core/pom.xml
+++ b/giraph-core/pom.xml
@@ -64,22 +64,6 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <configLocation>checkstyle.xml</configLocation>
- <enableRulesSummary>false</enableRulesSummary>
- <headerLocation>license-header.txt</headerLocation>
- <failOnViolation>true</failOnViolation>
- <includeTestSourceDirectory>false</includeTestSourceDirectory>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-examples/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-examples/pom.xml b/giraph-examples/pom.xml
index 7a18711..3b6a08c 100644
--- a/giraph-examples/pom.xml
+++ b/giraph-examples/pom.xml
@@ -46,22 +46,6 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <configLocation>checkstyle.xml</configLocation>
- <enableRulesSummary>false</enableRulesSummary>
- <headerLocation>license-header.txt</headerLocation>
- <failOnViolation>true</failOnViolation>
- <includeTestSourceDirectory>false</includeTestSourceDirectory>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -107,20 +91,6 @@ under the License.
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
- <version>2.5.1</version>
- <configuration>
- <xmlOutput>true</xmlOutput>
- <findbugsXmlOutput>false</findbugsXmlOutput>
- <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-hbase/pom.xml b/giraph-hbase/pom.xml
index 8eafff1..e7aa420 100644
--- a/giraph-hbase/pom.xml
+++ b/giraph-hbase/pom.xml
@@ -44,22 +44,6 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <configLocation>${top.dir}/checkstyle.xml</configLocation>
- <headerLocation>${top.dir}/license-header.txt</headerLocation>
- <enableRulesSummary>false</enableRulesSummary>
- <failOnError>true</failOnError>
- <includeTestSourceDirectory>false</includeTestSourceDirectory>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -81,20 +65,6 @@ under the License.
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
- <version>2.5.1</version>
- <configuration>
- <xmlOutput>true</xmlOutput>
- <findbugsXmlOutput>false</findbugsXmlOutput>
- <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hcatalog/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-hcatalog/pom.xml b/giraph-hcatalog/pom.xml
index 73359e3..ee5710a 100644
--- a/giraph-hcatalog/pom.xml
+++ b/giraph-hcatalog/pom.xml
@@ -44,22 +44,6 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <configLocation>${top.dir}/checkstyle.xml</configLocation>
- <headerLocation>${top.dir}/license-header.txt</headerLocation>
- <enableRulesSummary>false</enableRulesSummary>
- <failOnError>true</failOnError>
- <includeTestSourceDirectory>false</includeTestSourceDirectory>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -81,20 +65,6 @@ under the License.
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
- <version>2.5.1</version>
- <configuration>
- <xmlOutput>true</xmlOutput>
- <findbugsXmlOutput>false</findbugsXmlOutput>
- <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-hive/pom.xml b/giraph-hive/pom.xml
index 84b7c91..cc16ca2 100644
--- a/giraph-hive/pom.xml
+++ b/giraph-hive/pom.xml
@@ -44,22 +44,6 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <configLocation>${top.dir}/checkstyle.xml</configLocation>
- <headerLocation>${top.dir}/license-header.txt</headerLocation>
- <enableRulesSummary>false</enableRulesSummary>
- <failOnError>true</failOnError>
- <includeTestSourceDirectory>false</includeTestSourceDirectory>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -81,20 +65,6 @@ under the License.
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
- <version>2.5.1</version>
- <configuration>
- <xmlOutput>true</xmlOutput>
- <findbugsXmlOutput>false</findbugsXmlOutput>
- <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java b/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
index d89bad3..eeea265 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
@@ -30,7 +30,8 @@ import org.apache.giraph.hive.common.HiveProfiles;
import org.apache.giraph.hive.input.edge.HiveEdgeInputFormat;
import org.apache.giraph.hive.input.edge.HiveEdgeReader;
import org.apache.giraph.hive.input.edge.HiveToEdge;
-import org.apache.giraph.hive.input.vertex.HiveToVertex;
+import org.apache.giraph.hive.input.vertex.HiveToVertexValue;
+import org.apache.giraph.hive.input.vertex.HiveToVertexEdges;
import org.apache.giraph.hive.input.vertex.HiveVertexInputFormat;
import org.apache.giraph.hive.input.vertex.HiveVertexReader;
import org.apache.giraph.hive.output.HiveVertexOutputFormat;
@@ -77,7 +78,9 @@ public class HiveGiraphRunner implements Tool {
private Class<? extends Vertex> vertexClass;
/** Vertex creator from hive records. */
- private Class<? extends HiveToVertex> hiveToVertexClass;
+ private Class<? extends HiveToVertexValue> hiveToVertexClass;
+ /** Vertex edges creator from hive records. */
+ private Class<? extends HiveToVertexEdges> hiveToVertexEdgesClass;
/** hive vertex input information */
private final HiveInputDescription hiveVertexInputDescription;
@@ -104,58 +107,40 @@ public class HiveGiraphRunner implements Tool {
hiveOutputDescription = new HiveOutputDescription();
}
- /**
- * Get Vertex class used.
- *
- * @return Vertex class
- */
public Class<? extends Vertex> getVertexClass() {
return vertexClass;
}
- /**
- * Set Vertex class to use
- *
- * @param vertexClass Vertex class
- */
public void setVertexClass(Class<? extends Vertex> vertexClass) {
this.vertexClass = vertexClass;
}
- /**
- * Get hive vertex input description
- *
- * @return HiveInputDescription
- */
public HiveInputDescription getHiveVertexInputDescription() {
return hiveVertexInputDescription;
}
- /**
- * Get hive output description
- *
- * @return HiveOutputDescription
- */
public HiveOutputDescription getHiveOutputDescription() {
return hiveOutputDescription;
}
- /**
- * Get hive input description
- *
- * @return HiveInputDescription
- */
public HiveInputDescription getHiveEdgeInputDescription() {
return hiveEdgeInputDescription;
}
+ public Class<? extends HiveToVertexValue> getHiveToVertexValueClass() {
+ return hiveToVertexClass;
+ }
+
/**
- * Get HiveVertexCreator used with HiveVertexInputFormat
+ * Set HiveVertexCreator used with HiveVertexInputFormat
*
- * @return HiveVertexCreator
+ * @param hiveToVertexClass HiveVertexCreator
*/
- public Class<? extends HiveToVertex> getHiveToVertexClass() {
- return hiveToVertexClass;
+ public void setHiveToVertexValueClass(
+ Class<? extends HiveToVertexValue> hiveToVertexClass) {
+ this.hiveToVertexClass = hiveToVertexClass;
+ conf.setClass(HiveVertexReader.HIVE_TO_VERTEX_KEY, hiveToVertexClass,
+ HiveToVertexValue.class);
}
/**
@@ -163,37 +148,35 @@ public class HiveGiraphRunner implements Tool {
*
* @return true if vertex input enabled (HiveVertexCreator is set).
*/
- public boolean hasVertexInput() {
+ public boolean hasVertexValueInput() {
return hiveToVertexClass != null;
}
- /**
- * Set class used to convert hive records to Vertexes.
- *
- * @param hiveToVertexValueClass HiveToVertex class
- */
- public void setHiveToVertexValueClass(
- Class<? extends HiveToVertex> hiveToVertexValueClass) {
- setHiveToVertexClass(hiveToVertexValueClass);
+ public Class<? extends HiveToVertexEdges> getHiveToVertexEdgesClass() {
+ return hiveToVertexEdgesClass;
}
/**
- * Set HiveVertexCreator used with HiveVertexInputFormat
+ * Whether we have a class for reading per-vertex edges from Hive.
*
- * @param hiveToVertexClass HiveVertexCreator
+ * @return true if user set class for reading vertex edges.
*/
- public void setHiveToVertexClass(
- Class<? extends HiveToVertex> hiveToVertexClass) {
- this.hiveToVertexClass = hiveToVertexClass;
- conf.setClass(HiveVertexReader.HIVE_TO_VERTEX_KEY, hiveToVertexClass,
- HiveToVertex.class);
+ public boolean hasHiveToVertexEdgesClass() {
+ return hiveToVertexEdgesClass != null;
}
/**
- * Get HiveEdgeCreator used with HiveEdgeInputFormat
+ * Set class for reading per-vertex edges from hive.
*
- * @return HiveEdgeCreator
+ * @param klass Class to use
*/
+ public void setHiveToVertexEdgesClass(
+ Class<? extends HiveToVertexEdges> klass) {
+ this.hiveToVertexEdgesClass = klass;
+ conf.setClass(HiveVertexReader.HIVE_TO_VERTEX_EDGES_KEY, klass,
+ HiveToVertexEdges.class);
+ }
+
public Class<? extends HiveToEdge> getHiveToEdgeClass() {
return hiveToEdgeClass;
}
@@ -214,15 +197,10 @@ public class HiveGiraphRunner implements Tool {
*/
public void setHiveToEdgeClass(Class<? extends HiveToEdge> hiveToEdgeClass) {
this.hiveToEdgeClass = hiveToEdgeClass;
- conf.setClass(HiveEdgeReader.EDGE_CREATOR_KEY, hiveToEdgeClass,
+ conf.setClass(HiveEdgeReader.HIVE_TO_EDGE_KEY, hiveToEdgeClass,
HiveToEdge.class);
}
- /**
- * Get class used to write vertices to Hive.
- *
- * @return class for writing vertices to Hive
- */
public Class<? extends VertexToHive> getVertexToHiveClass() {
return vertexToHiveClass;
}
@@ -386,7 +364,15 @@ public class HiveGiraphRunner implements Tool {
String hiveToVertexClassStr = cmdln.getOptionValue("hiveToVertexClass");
if (hiveToVertexClassStr != null) {
- hiveToVertexClass = findClass(hiveToVertexClassStr, HiveToVertex.class);
+ setHiveToVertexValueClass(findClass(hiveToVertexClassStr,
+ HiveToVertexValue.class));
+ }
+
+ String hiveToVertexEdgesClassStr =
+ cmdln.getOptionValue("hiveToVertexEdgesClass");
+ if (hiveToVertexEdgesClassStr != null) {
+ setHiveToVertexEdgesClass(findClass(hiveToVertexEdgesClassStr,
+ HiveToVertexEdges.class));
}
String hiveToEdgeClassStr = cmdln.getOptionValue("hiveToEdgeClass");
@@ -396,26 +382,26 @@ public class HiveGiraphRunner implements Tool {
String vertexToHiveClassStr = cmdln.getOptionValue("vertexToHiveClass");
if (vertexToHiveClassStr != null) {
- vertexToHiveClass = findClass(vertexToHiveClassStr, VertexToHive.class);
+ setVertexToHiveClass(findClass(vertexToHiveClassStr, VertexToHive.class));
}
if (cmdln.hasOption("skipOutput")) {
skipOutput = true;
}
-// if (hiveToVertexClass == null && hiveToEdgeClass == null) {
-// throw new IllegalArgumentException(
-// "Need at least one of Giraph " +
-// HiveToVertex.class.getSimpleName() +
-// " class name (-hiveToVertexClass) and " +
-// HiveToEdge.class.getSimpleName() +
-// " class name (-hiveToEdgeClass)");
-// }
-// if (vertexToHiveClass == null && !skipOutput) {
-// throw new IllegalArgumentException(
-// "Need the Giraph " + VertexToHive.class.getSimpleName() +
-// " class name (-vertexToHiveClass) to use");
-// }
+ if (hiveToVertexClass == null && hiveToEdgeClass == null) {
+ throw new IllegalArgumentException(
+ "Need at least one of Giraph " +
+ HiveToVertexValue.class.getSimpleName() +
+ " class name (-hiveToVertexClass) and " +
+ HiveToEdge.class.getSimpleName() +
+ " class name (-hiveToEdgeClass)");
+ }
+ if (vertexToHiveClass == null && !skipOutput) {
+ throw new IllegalArgumentException(
+ "Need the Giraph " + VertexToHive.class.getSimpleName() +
+ " class name (-vertexToHiveClass) to use");
+ }
String workersStr = cmdln.getOptionValue("workers");
if (workersStr == null) {
throw new IllegalArgumentException(
@@ -536,7 +522,12 @@ public class HiveGiraphRunner implements Tool {
// Vertex input settings
if (hiveToVertexClass == null) {
options.addOption(null, "hiveToVertexClass", true,
- "Giraph " + HiveToVertex.class.getSimpleName() + " class to use");
+ "Giraph " + HiveToVertexValue.class.getSimpleName() + " class to use");
+ }
+ if (hiveToVertexEdgesClass == null) {
+ options.addOption(null, "hiveToVertexEdgesClass", true,
+ "Giraph " + HiveToVertexEdges.class.getSimpleName() +
+ " class to use");
}
options.addOption("vi", "vertexInputTable", true,
"Vertex input table name");
@@ -662,6 +653,10 @@ public class HiveGiraphRunner implements Tool {
LOG.info(LOG_PREFIX + "-hiveToVertexClass=" +
hiveToVertexClass.getCanonicalName());
}
+ if (hiveToVertexEdgesClass != null) {
+ LOG.info(LOG_PREFIX + "-hiveToVertexEdgesClass=" +
+ hiveToVertexEdgesClass.getCanonicalName());
+ }
LOG.info(LOG_PREFIX + "-vertexInputFormatClass=" +
classes.getVertexInputFormatClass().getCanonicalName());
logInputDesc(hiveVertexInputDescription, "vertex");
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
new file mode 100644
index 0000000..c418085
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.input.edge;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+import com.facebook.giraph.hive.HiveTableSchemaAware;
+
+/**
+ * Base class for HiveToEdge implementations
+ *
+ * @param <I> Vertex ID
+ * @param <V> Vertex Value
+ * @param <E> Edge Value
+ * @param <M> Message Value
+ */
+public abstract class AbstractHiveToEdge<I extends WritableComparable,
+ V extends Writable, E extends Writable, M extends Writable> implements
+ HiveTableSchemaAware,
+ HiveToEdge<I, E>,
+ ImmutableClassesGiraphConfigurable<I, V, E, M> { }
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveEdgeReader.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveEdgeReader.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveEdgeReader.java
index 0906996..e0c502c 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveEdgeReader.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveEdgeReader.java
@@ -18,20 +18,21 @@
package org.apache.giraph.hive.input.edge;
-import com.facebook.giraph.hive.HiveRecord;
-import com.facebook.giraph.hive.HiveTableSchema;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.io.EdgeReader;
-import org.apache.giraph.utils.ReflectionUtils;
import org.apache.giraph.edge.Edge;
-import org.apache.giraph.edge.EdgeFactory;
import org.apache.giraph.edge.MutableEdge;
+import org.apache.giraph.io.EdgeReader;
+import org.apache.giraph.utils.ReflectionUtils;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import com.facebook.giraph.hive.HiveRecord;
+import com.facebook.giraph.hive.HiveTableSchema;
+import com.facebook.giraph.hive.HiveTableSchemas;
+
import java.io.IOException;
/**
@@ -43,7 +44,7 @@ import java.io.IOException;
public class HiveEdgeReader<I extends WritableComparable, E extends Writable>
implements EdgeReader<I, E> {
/** Configuration key for edge creator class */
- public static final String EDGE_CREATOR_KEY = "giraph.hive.to.edge.class";
+ public static final String HIVE_TO_EDGE_KEY = "giraph.hive.to.edge.class";
/** Configuration key for whether to reuse edge */
public static final String REUSE_EDGE_KEY = "giraph.hive.reuse.edge";
@@ -126,13 +127,13 @@ public class HiveEdgeReader<I extends WritableComparable, E extends Writable>
* @throws IOException if anything goes wrong reading from Configuration
*/
private void instantiateHiveToEdgeFromConf() throws IOException {
- Class<? extends HiveToEdge> klass = conf.getClass(EDGE_CREATOR_KEY,
+ Class<? extends HiveToEdge> klass = conf.getClass(HIVE_TO_EDGE_KEY,
null, HiveToEdge.class);
if (klass == null) {
- throw new IOException(EDGE_CREATOR_KEY + " not set in conf");
+ throw new IOException(HIVE_TO_EDGE_KEY + " not set in conf");
}
hiveToEdge = ReflectionUtils.newInstance(klass, conf);
- hiveToEdge.setTableSchema(tableSchema);
+ HiveTableSchemas.configure(hiveToEdge, tableSchema);
}
@Override
@@ -159,14 +160,12 @@ public class HiveEdgeReader<I extends WritableComparable, E extends Writable>
public Edge<I, E> getCurrentEdge() throws IOException,
InterruptedException {
HiveRecord record = hiveRecordReader.getCurrentValue();
- I targetId = hiveToEdge.getTargetVertexId(record);
- E edgeValue = hiveToEdge.getEdgeValue(record);
- if (edgeToReuse == null) {
- return EdgeFactory.create(targetId, edgeValue);
- } else {
- edgeToReuse.setTargetVertexId(targetId);
- edgeToReuse.setValue(edgeValue);
- return edgeToReuse;
+ MutableEdge<I, E> edge = edgeToReuse;
+ if (edge == null) {
+ edge = conf.createMutableEdge();
}
+ edge.setValue(hiveToEdge.getEdgeValue(record));
+ edge.setTargetVertexId(hiveToEdge.getTargetVertexId(record));
+ return edge;
}
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveToEdge.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveToEdge.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveToEdge.java
index e800321..2205b82 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveToEdge.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveToEdge.java
@@ -22,16 +22,14 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import com.facebook.giraph.hive.HiveReadableRecord;
-import com.facebook.giraph.hive.HiveTableSchemaAware;
/**
- * An interface used to create edges from Hive records
+ * An interface used to create edges from Hive records.
*
* @param <I> Vertex ID
* @param <E> Edge Value
*/
-public interface HiveToEdge<I extends WritableComparable, E extends Writable>
- extends HiveTableSchemaAware {
+public interface HiveToEdge<I extends WritableComparable, E extends Writable> {
/**
* Read source vertex ID from Hive record
*
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexEdges.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexEdges.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexEdges.java
new file mode 100644
index 0000000..746e9e3
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexEdges.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.input.vertex;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+import com.facebook.giraph.hive.HiveTableSchemaAware;
+
+/**
+ * Base class for HiveToVertexEdges implementations
+ *
+ * @param <I> Vertex ID
+ * @param <V> Vertex Value
+ * @param <E> Edge Value
+ * @param <M> Message Value
+ */
+public abstract class AbstractHiveToVertexEdges<I extends WritableComparable,
+ V extends Writable, E extends Writable, M extends Writable> implements
+ HiveTableSchemaAware,
+ HiveToVertexEdges<I, E>,
+ ImmutableClassesGiraphConfigurable<I, V, E, M> { }
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexValue.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexValue.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexValue.java
new file mode 100644
index 0000000..9636d2a
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexValue.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.input.vertex;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+import com.facebook.giraph.hive.HiveTableSchemaAware;
+
+/**
+ * Base class for HiveToVertex implementations
+ *
+ * @param <I> Vertex ID
+ * @param <V> Vertex Value
+ * @param <E> Edge Value
+ * @param <M> Message Value
+ */
+public abstract class AbstractHiveToVertexValue<I extends WritableComparable,
+ V extends Writable, E extends Writable, M extends Writable> implements
+ HiveTableSchemaAware, HiveToVertexValue<I, V>,
+ ImmutableClassesGiraphConfigurable<I, V, E, M> { }
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertex.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertex.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertex.java
deleted file mode 100644
index 7c84ff1..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertex.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive.input.vertex;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import com.facebook.giraph.hive.HiveReadableRecord;
-import com.facebook.giraph.hive.HiveTableSchemaAware;
-
-/**
- * Interface for creating vertices from a Hive record.
- * Also used for reading vertex values.
- *
- * @param <I> Vertex ID
- * @param <V> Vertex Value
- * @param <E> Edge Value
- */
-public interface HiveToVertex<I extends WritableComparable,
- V extends Writable, E extends Writable> extends HiveTableSchemaAware {
- /**
- * Fill the Vertex from the HiveRecord given.
- *
- * @param record HiveRecord to read from.
- * @param vertex Vertex to fill.
- */
- void fillVertex(HiveReadableRecord record, Vertex<I, V, E, ?> vertex);
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexEdges.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexEdges.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexEdges.java
new file mode 100644
index 0000000..cf7ea33
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexEdges.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.hive.input.vertex;
+
+import org.apache.giraph.edge.Edge;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+import com.facebook.giraph.hive.HiveReadableRecord;
+
+import java.util.Collections;
+
+/**
+ * Interface for creating edges for a vertex from a Hive record.
+ * Used with HiveToVertex if you want to also read edges per vertex, as opposed
+ * to using {@link org.apache.giraph.hive.input.edge.HiveEdgeInputFormat}
+ *
+ * @param <I> Vertex ID
+ * @param <E> extends Writable
+ */
+public interface HiveToVertexEdges<I extends WritableComparable,
+ E extends Writable> {
+ /**
+ * Read Vertex's edges from the HiveRecord given.
+ *
+ * @param record HiveRecord to read from.
+ * @return iterable of edges
+ */
+ Iterable<Edge<I, E>> getEdges(HiveReadableRecord record);
+
+ /**
+ * Default implementation that returns an empty list of edges.
+ */
+ public class Empty implements HiveToVertexEdges {
+ /** Singleton */
+ private static final Empty INSTANCE = new Empty();
+
+ /** Don't construct */
+ private Empty() { }
+
+ /**
+ * Get singleton instance
+ * @return Empty
+ */
+ public static Empty get() { return INSTANCE; }
+
+ @Override public Iterable getEdges(HiveReadableRecord record) {
+ return Collections.emptyList();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexValue.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexValue.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexValue.java
new file mode 100644
index 0000000..593eb9a
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexValue.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.hive.input.vertex;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+import com.facebook.giraph.hive.HiveReadableRecord;
+
+/**
+ * Interface for creating vertices from a Hive record.
+ *
+ * @param <I> Vertex ID
+ * @param <V> Vertex Value
+ */
+public interface HiveToVertexValue<I extends WritableComparable,
+ V extends Writable> {
+ /**
+ * Read the Vertex's ID from the HiveRecord given.
+ *
+ * @param record HiveRecord to read from.
+ * @return Vertex ID
+ */
+ I getVertexId(HiveReadableRecord record);
+
+ /**
+ * Read the Vertex's Value from the HiveRecord given.
+ *
+ * @param record HiveRecord to read from.
+ * @return Vertex Value
+ */
+ V getVertexValue(HiveReadableRecord record);
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveVertexReader.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveVertexReader.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveVertexReader.java
index b0f348b..2311e72 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveVertexReader.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveVertexReader.java
@@ -19,9 +19,10 @@
package org.apache.giraph.hive.input.vertex;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.edge.Edge;
+import org.apache.giraph.graph.Vertex;
import org.apache.giraph.io.VertexReader;
import org.apache.giraph.utils.ReflectionUtils;
-import org.apache.giraph.graph.Vertex;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -31,6 +32,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
import com.facebook.giraph.hive.HiveRecord;
import com.facebook.giraph.hive.HiveTableSchema;
import com.facebook.giraph.hive.HiveTableSchemaAware;
+import com.facebook.giraph.hive.HiveTableSchemas;
import java.io.IOException;
@@ -45,8 +47,14 @@ import java.io.IOException;
public class HiveVertexReader<I extends WritableComparable,
V extends Writable, E extends Writable, M extends Writable>
implements VertexReader<I, V, E, M>, HiveTableSchemaAware {
- /** Key in Configuration for HiveToVertex class */
- public static final String HIVE_TO_VERTEX_KEY = "giraph.hive.to.vertex.class";
+ /** Configuration key for {@link HiveToVertexValue} class */
+ public static final String HIVE_TO_VERTEX_KEY =
+ "giraph.hive.to.vertex.value.class";
+ /** Configuration key for {@link HiveToVertexEdges} class */
+ public static final String HIVE_TO_VERTEX_EDGES_KEY =
+ "giraph.hive.to.vertex.edges.class";
+ /** Configuration key for whether to reuse vertex */
+ public static final String REUSE_VERTEX_KEY = "giraph.hive.reuse.vertex";
/** Underlying Hive RecordReader used */
private RecordReader<WritableComparable, HiveRecord> hiveRecordReader;
@@ -57,7 +65,15 @@ public class HiveVertexReader<I extends WritableComparable,
private ImmutableClassesGiraphConfiguration<I, V, E, M> conf;
/** User class to create vertices from a HiveRecord */
- private HiveToVertex<I, V, E> hiveToVertex;
+ private HiveToVertexValue<I, V> hiveToVertexValue;
+ /** User class to create vertex edges from HiveRecord - optional */
+ private HiveToVertexEdges<I, E> hiveToVertexEdges;
+
+ /**
+ * If we are reusing vertices this will be the single vertex to read into.
+ * Otherwise if it's null we will create a new vertex each time.
+ */
+ private Vertex<I, V, E, M> vertexToReuse = null;
/**
* Get underlying Hive record reader used.
@@ -103,21 +119,40 @@ public class HiveVertexReader<I extends WritableComparable,
hiveRecordReader.initialize(inputSplit, context);
conf = new ImmutableClassesGiraphConfiguration(context.getConfiguration());
instantiateHiveToVertexFromConf();
+ instantiateHiveToVertexEdgesFromConf();
+ if (conf.getBoolean(REUSE_VERTEX_KEY, false)) {
+ vertexToReuse = conf.createVertex();
+ }
}
/**
- * Retrieve the user's HiveVertexCreator from our configuration.
+ * Retrieve the user's HiveToVertex from our configuration.
*
* @throws IOException if anything goes wrong reading from Configuration.
*/
private void instantiateHiveToVertexFromConf() throws IOException {
- Class<? extends HiveToVertex> klass = conf.getClass(HIVE_TO_VERTEX_KEY,
- null, HiveToVertex.class);
+ Class<? extends HiveToVertexValue> klass = conf.getClass(HIVE_TO_VERTEX_KEY,
+ null, HiveToVertexValue.class);
if (klass == null) {
throw new IOException(HIVE_TO_VERTEX_KEY + " not set in conf");
}
- hiveToVertex = ReflectionUtils.newInstance(klass, conf);
- hiveToVertex.setTableSchema(tableSchema);
+ hiveToVertexValue = ReflectionUtils.newInstance(klass, conf);
+ HiveTableSchemas.configure(hiveToVertexValue, tableSchema);
+ }
+
+ /**
+ * Retrieve the user's HiveToVertexEdges from our configuration. This class
+ * is optional. If not specified will just use HiveToVertexEdges.Empty.
+ */
+ private void instantiateHiveToVertexEdgesFromConf() {
+ Class<? extends HiveToVertexEdges> klass = conf.getClass(
+ HIVE_TO_VERTEX_EDGES_KEY, null, HiveToVertexEdges.class);
+ if (klass == null) {
+ hiveToVertexEdges = HiveToVertexEdges.Empty.get();
+ } else {
+ hiveToVertexEdges = ReflectionUtils.newInstance(klass, conf);
+ }
+ HiveTableSchemas.configure(hiveToVertexEdges, tableSchema);
}
@Override
@@ -139,8 +174,14 @@ public class HiveVertexReader<I extends WritableComparable,
public final Vertex<I, V, E, M> getCurrentVertex()
throws IOException, InterruptedException {
HiveRecord hiveRecord = hiveRecordReader.getCurrentValue();
- Vertex vertex = conf.createVertex();
- hiveToVertex.fillVertex(hiveRecord, vertex);
+ Vertex<I, V, E, M> vertex = vertexToReuse;
+ if (vertex == null) {
+ vertex = conf.createVertex();
+ }
+ I id = hiveToVertexValue.getVertexId(hiveRecord);
+ V value = hiveToVertexValue.getVertexValue(hiveRecord);
+ Iterable<Edge<I, E>> edges = hiveToVertexEdges.getEdges(hiveRecord);
+ vertex.initialize(id, value, edges);
return vertex;
}
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java
new file mode 100644
index 0000000..f2f2267
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.output;
+
+import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+
+import com.facebook.giraph.hive.HiveTableSchemaAware;
+
+/**
+ * Base class for HiveToVertexEdges implementations
+ *
+ * @param <I> Vertex ID
+ * @param <V> Vertex Value
+ * @param <E> Edge Value
+ * @param <M> Message Value
+ */
+public abstract class AbstractVertexToHive<I extends WritableComparable,
+ V extends Writable, E extends Writable, M extends Writable> implements
+ HiveTableSchemaAware,
+ VertexToHive<I, V, E>,
+ ImmutableClassesGiraphConfigurable<I, V, E, M> { }
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java
index 32bae15..47d096b 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java
@@ -19,9 +19,9 @@
package org.apache.giraph.hive.output;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.graph.Vertex;
import org.apache.giraph.io.VertexWriter;
import org.apache.giraph.utils.ReflectionUtils;
-import org.apache.giraph.graph.Vertex;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
@@ -31,6 +31,7 @@ import org.apache.log4j.Logger;
import com.facebook.giraph.hive.HiveRecord;
import com.facebook.giraph.hive.HiveTableSchema;
+import com.facebook.giraph.hive.HiveTableSchemas;
import com.facebook.giraph.hive.impl.HiveApiRecord;
import java.io.IOException;
@@ -117,7 +118,7 @@ public class HiveVertexWriter<I extends WritableComparable, V extends Writable,
throw new IOException(VERTEX_TO_HIVE_KEY + " not set in conf");
}
vertexToHive = ReflectionUtils.newInstance(klass, conf);
- vertexToHive.setTableSchema(tableSchema);
+ HiveTableSchemas.configure(vertexToHive, tableSchema);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java
index 434d804..6d323bd 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java
@@ -22,7 +22,6 @@ import org.apache.giraph.graph.Vertex;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
-import com.facebook.giraph.hive.HiveTableSchemaAware;
import com.facebook.giraph.hive.HiveWritableRecord;
/**
@@ -33,7 +32,7 @@ import com.facebook.giraph.hive.HiveWritableRecord;
* @param <E> Edge Value
*/
public interface VertexToHive<I extends WritableComparable, V extends Writable,
- E extends Writable> extends HiveTableSchemaAware {
+ E extends Writable> {
/**
* Fill the HiveRecord from the Vertex given.
*
http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a65208d..509831b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -301,6 +301,27 @@ under the License.
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>2.10</version>
+ <configuration>
+ <configLocation>${top.dir}/checkstyle.xml</configLocation>
+ <consoleOutput>true</consoleOutput>
+ <enableRulesSummary>false</enableRulesSummary>
+ <headerLocation>${top.dir}/license-header.txt</headerLocation>
+ <failOnViolation>true</failOnViolation>
+ <includeTestSourceDirectory>false</includeTestSourceDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.0</version>
<configuration>
@@ -484,6 +505,24 @@ under the License.
</configuration>
</plugin>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <version>2.5.2</version>
+ <configuration>
+ <xmlOutput>true</xmlOutput>
+ <findbugsXmlOutput>false</findbugsXmlOutput>
+ <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>munge-maven-plugin</artifactId>
<version>1.0</version>
@@ -837,7 +876,7 @@ under the License.
<dependency>
<groupId>com.facebook.giraph.hive</groupId>
<artifactId>hive-io-experimental</artifactId>
- <version>0.1</version>
+ <version>0.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>