You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2013/03/25 05:36:32 UTC
[2/2] git commit: fix schema issue
fix schema issue
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/8fe2a03c
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/8fe2a03c
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/8fe2a03c
Branch: refs/heads/perf
Commit: 8fe2a03cf964158270ffba7e69777230968b3445
Parents: 9a7c55f
Author: Nitay Joffe <ni...@apache.org>
Authored: Mon Mar 25 00:36:26 2013 -0400
Committer: Nitay Joffe <ni...@apache.org>
Committed: Mon Mar 25 00:36:26 2013 -0400
----------------------------------------------------------------------
.../org/apache/giraph/hive/HiveGiraphRunner.java | 22 +++++++++++----
.../apache/giraph/hive/common/HiveProfiles.java | 3 --
pom.xml | 2 +-
3 files changed, 17 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/8fe2a03c/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java b/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
index 4b6cab9..5233709 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
@@ -26,19 +26,18 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.giraph.conf.GiraphClasses;
import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.hive.common.HiveProfiles;
+import org.apache.giraph.graph.Vertex;
import org.apache.giraph.hive.input.edge.HiveEdgeInputFormat;
import org.apache.giraph.hive.input.edge.HiveEdgeReader;
import org.apache.giraph.hive.input.edge.HiveToEdge;
-import org.apache.giraph.hive.input.vertex.HiveToVertexValue;
import org.apache.giraph.hive.input.vertex.HiveToVertexEdges;
+import org.apache.giraph.hive.input.vertex.HiveToVertexValue;
import org.apache.giraph.hive.input.vertex.HiveVertexInputFormat;
import org.apache.giraph.hive.input.vertex.HiveVertexReader;
import org.apache.giraph.hive.output.HiveVertexOutputFormat;
import org.apache.giraph.hive.output.HiveVertexWriter;
import org.apache.giraph.hive.output.VertexToHive;
import org.apache.giraph.job.GiraphJob;
-import org.apache.giraph.graph.Vertex;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.util.Tool;
@@ -50,6 +49,7 @@ import com.facebook.giraph.hive.input.HiveApiInputFormat;
import com.facebook.giraph.hive.input.HiveInputDescription;
import com.facebook.giraph.hive.output.HiveApiOutputFormat;
import com.facebook.giraph.hive.output.HiveOutputDescription;
+import com.facebook.giraph.hive.schema.HiveTableSchemas;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -60,6 +60,10 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
+import static org.apache.giraph.hive.common.HiveProfiles.EDGE_INPUT_PROFILE_ID;
+import static org.apache.giraph.hive.common.HiveProfiles.VERTEX_INPUT_PROFILE_ID;
+import static org.apache.giraph.hive.common.HiveProfiles.VERTEX_OUTPUT_PROFILE_ID;
+
/**
* Hive Giraph Runner
*/
@@ -276,14 +280,18 @@ public class HiveGiraphRunner implements Tool {
private void setupHiveInputs(GiraphConfiguration conf) throws TException {
if (hiveToVertexClass != null) {
HiveApiInputFormat.setProfileInputDesc(conf, hiveVertexInputDescription,
- HiveProfiles.VERTEX_INPUT_PROFILE_ID);
+ VERTEX_INPUT_PROFILE_ID);
conf.setVertexInputFormatClass(HiveVertexInputFormat.class);
+ HiveTableSchemas.put(conf, VERTEX_INPUT_PROFILE_ID,
+ hiveVertexInputDescription.hiveTableName());
}
if (hiveToEdgeClass != null) {
HiveApiInputFormat.setProfileInputDesc(conf, hiveEdgeInputDescription,
- HiveProfiles.EDGE_INPUT_PROFILE_ID);
+ EDGE_INPUT_PROFILE_ID);
conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
+ HiveTableSchemas.put(conf, EDGE_INPUT_PROFILE_ID,
+ hiveEdgeInputDescription.hiveTableName());
}
}
@@ -298,8 +306,10 @@ public class HiveGiraphRunner implements Tool {
LOG.warn("run: Warning - Output will be skipped!");
} else if (vertexToHiveClass != null) {
HiveApiOutputFormat.initProfile(conf, hiveOutputDescription,
- HiveProfiles.VERTEX_OUTPUT_PROFILE_ID);
+ VERTEX_OUTPUT_PROFILE_ID);
conf.setVertexOutputFormatClass(HiveVertexOutputFormat.class);
+ HiveTableSchemas.put(conf, VERTEX_OUTPUT_PROFILE_ID,
+ hiveOutputDescription.hiveTableName());
} else {
LOG.fatal("output requested but " + VertexToHive.class.getSimpleName() +
" not set");
http://git-wip-us.apache.org/repos/asf/giraph/blob/8fe2a03c/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveProfiles.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveProfiles.java b/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveProfiles.java
index b0ddc48..892d443 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveProfiles.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveProfiles.java
@@ -26,9 +26,6 @@ public class HiveProfiles {
public static final String VERTEX_INPUT_PROFILE_ID = "vertex_input_profile";
/** name of edge input profile */
public static final String EDGE_INPUT_PROFILE_ID = "edge_input_profile";
- /** name of vertex value input profile */
- public static final String VERTEX_VALUE_INPUT_PROFILE_ID =
- "vertex_value_input_profile";
/** Name of vertex output profile */
public static final String VERTEX_OUTPUT_PROFILE_ID = "vertex_output_profile";
http://git-wip-us.apache.org/repos/asf/giraph/blob/8fe2a03c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e576e4b..d94ce7a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -881,7 +881,7 @@ under the License.
<dependency>
<groupId>com.facebook.giraph.hive</groupId>
<artifactId>hive-io-experimental</artifactId>
- <version>0.4-SNAPSHOT</version>
+ <version>0.5-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>