You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/13 23:28:17 UTC
incubator-tinkerpop git commit: Fixed a bug where the inputformat and
outputformat are different after a VertexProgram job execution relative to a
MapReduce job.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master a65b27736 -> be80e558f
Fixed a bug where the inputformat and outputformat are different after a VertexProgram job execution relative to a MapReduce job.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/be80e558
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/be80e558
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/be80e558
Branch: refs/heads/master
Commit: be80e558f116b03d93529d4972759dd153b939a6
Parents: a65b277
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri Mar 13 16:28:06 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri Mar 13 16:28:15 2015 -0600
----------------------------------------------------------------------
data/tinkerpop-crew-vertices.ldjson | 6 ++
.../io/graphson/tinkerpop-crew-vertices.ldjson | 6 ++
.../graphson/tinkerpop-modern-vertices.ldjson | 6 ++
.../process/computer/util/MapReduceHelper.java | 10 +++-
.../hadoop/structure/io/InputOutputHelper.java | 60 ++++++++++++++++++++
.../io/graphson/GraphSONInputFormat.java | 2 +-
.../io/graphson/GraphSONOutputFormat.java | 5 +-
.../structure/io/gryo/GryoOutputFormat.java | 4 +-
.../structure/io/script/ScriptInputFormat.java | 2 +-
.../structure/io/script/ScriptOutputFormat.java | 4 +-
.../structure/io/script/ScriptRecordReader.java | 3 +-
.../gremlin/hadoop/HadoopGraphProvider.java | 17 ++++--
.../tinkergraph/structure/TinkerGraphTest.java | 13 ++++-
13 files changed, 118 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/data/tinkerpop-crew-vertices.ldjson
----------------------------------------------------------------------
diff --git a/data/tinkerpop-crew-vertices.ldjson b/data/tinkerpop-crew-vertices.ldjson
new file mode 100644
index 0000000..bb1f721
--- /dev/null
+++ b/data/tinkerpop-crew-vertices.ldjson
@@ -0,0 +1,6 @@
+{"inE":[],"outE":[{"inV":11,"inVLabel":"software","outVLabel":"person","id":16,"label":"uses","type":"edge","outV":1,"properties":{"skill":5}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":15,"label":"uses","type":"edge","outV":1,"properties":{"skill":4}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":13,"label":"develops","type":"edge","outV":1,"properties":{"since":2009}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":14,"label":"develops","type":"edge","outV":1,"properties":{"since":2010}}],"id":1,"label":"person","type":"vertex","properties":{"name":[{"id":0,"label":"name","value":"marko","properties":{}}],"location":[{"id":6,"label":"location","value":"san diego","properties":{"startTime":1997,"endTime":2001}},{"id":7,"label":"location","value":"santa cruz","properties":{"startTime":2001,"endTime":2004}},{"id":8,"label":"location","value":"brussels","properties":{"startTime":2004,"endTime":2005}},{"id":9,"label":"location","value":"santa fe","p
roperties":{"startTime":2005}}]}}
+{"inE":[],"outE":[{"inV":10,"inVLabel":"software","outVLabel":"person","id":19,"label":"uses","type":"edge","outV":7,"properties":{"skill":5}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":20,"label":"uses","type":"edge","outV":7,"properties":{"skill":4}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":17,"label":"develops","type":"edge","outV":7,"properties":{"since":2010}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":18,"label":"develops","type":"edge","outV":7,"properties":{"since":2011}}],"id":7,"label":"person","type":"vertex","properties":{"name":[{"id":1,"label":"name","value":"stephen","properties":{}}],"location":[{"id":10,"label":"location","value":"centreville","properties":{"startTime":1990,"endTime":2000}},{"id":11,"label":"location","value":"dulles","properties":{"startTime":2000,"endTime":2006}},{"id":12,"label":"location","value":"purcellville","properties":{"startTime":2006}}]}}
+{"inE":[],"outE":[{"inV":10,"inVLabel":"software","outVLabel":"person","id":22,"label":"uses","type":"edge","outV":8,"properties":{"skill":3}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":23,"label":"uses","type":"edge","outV":8,"properties":{"skill":3}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":21,"label":"develops","type":"edge","outV":8,"properties":{"since":2012}}],"id":8,"label":"person","type":"vertex","properties":{"name":[{"id":2,"label":"name","value":"matthias","properties":{}}],"location":[{"id":13,"label":"location","value":"bremen","properties":{"startTime":2004,"endTime":2007}},{"id":14,"label":"location","value":"baltimore","properties":{"startTime":2007,"endTime":2011}},{"id":15,"label":"location","value":"oakland","properties":{"startTime":2011,"endTime":2014}},{"id":16,"label":"location","value":"seattle","properties":{"startTime":2014}}]}}
+{"inE":[],"outE":[{"inV":10,"inVLabel":"software","outVLabel":"person","id":24,"label":"uses","type":"edge","outV":9,"properties":{"skill":5}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":25,"label":"uses","type":"edge","outV":9,"properties":{"skill":3}}],"id":9,"label":"person","type":"vertex","properties":{"name":[{"id":3,"label":"name","value":"daniel","properties":{}}],"location":[{"id":17,"label":"location","value":"spremberg","properties":{"startTime":1982,"endTime":2005}},{"id":18,"label":"location","value":"kaiserslautern","properties":{"startTime":2005,"endTime":2009}},{"id":19,"label":"location","value":"aachen","properties":{"startTime":2009}}]}}
+{"inE":[{"inV":10,"inVLabel":"software","outVLabel":"person","id":19,"label":"uses","type":"edge","outV":7,"properties":{"skill":5}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":22,"label":"uses","type":"edge","outV":8,"properties":{"skill":3}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":24,"label":"uses","type":"edge","outV":9,"properties":{"skill":5}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":15,"label":"uses","type":"edge","outV":1,"properties":{"skill":4}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":17,"label":"develops","type":"edge","outV":7,"properties":{"since":2010}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":21,"label":"develops","type":"edge","outV":8,"properties":{"since":2012}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":13,"label":"develops","type":"edge","outV":1,"properties":{"since":2009}}],"outE":[{"inV":11,"inVLabel":"software","outVLabel":"software","id":26,"label":"traverses","
type":"edge","outV":10,"properties":{}}],"id":10,"label":"software","type":"vertex","properties":{"name":[{"id":4,"label":"name","value":"gremlin","properties":{}}]}}
+{"inE":[{"inV":11,"inVLabel":"software","outVLabel":"software","id":26,"label":"traverses","type":"edge","outV":10,"properties":{}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":16,"label":"uses","type":"edge","outV":1,"properties":{"skill":5}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":20,"label":"uses","type":"edge","outV":7,"properties":{"skill":4}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":23,"label":"uses","type":"edge","outV":8,"properties":{"skill":3}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":25,"label":"uses","type":"edge","outV":9,"properties":{"skill":3}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":18,"label":"develops","type":"edge","outV":7,"properties":{"since":2011}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":14,"label":"develops","type":"edge","outV":1,"properties":{"since":2010}}],"outE":[],"id":11,"label":"software","type":"vertex","properties":{"name":[{"id":5,"label":"name","v
alue":"tinkergraph","properties":{}}]}}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/graphson/tinkerpop-crew-vertices.ldjson
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/graphson/tinkerpop-crew-vertices.ldjson b/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/graphson/tinkerpop-crew-vertices.ldjson
new file mode 100644
index 0000000..bb1f721
--- /dev/null
+++ b/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/graphson/tinkerpop-crew-vertices.ldjson
@@ -0,0 +1,6 @@
+{"inE":[],"outE":[{"inV":11,"inVLabel":"software","outVLabel":"person","id":16,"label":"uses","type":"edge","outV":1,"properties":{"skill":5}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":15,"label":"uses","type":"edge","outV":1,"properties":{"skill":4}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":13,"label":"develops","type":"edge","outV":1,"properties":{"since":2009}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":14,"label":"develops","type":"edge","outV":1,"properties":{"since":2010}}],"id":1,"label":"person","type":"vertex","properties":{"name":[{"id":0,"label":"name","value":"marko","properties":{}}],"location":[{"id":6,"label":"location","value":"san diego","properties":{"startTime":1997,"endTime":2001}},{"id":7,"label":"location","value":"santa cruz","properties":{"startTime":2001,"endTime":2004}},{"id":8,"label":"location","value":"brussels","properties":{"startTime":2004,"endTime":2005}},{"id":9,"label":"location","value":"santa fe","p
roperties":{"startTime":2005}}]}}
+{"inE":[],"outE":[{"inV":10,"inVLabel":"software","outVLabel":"person","id":19,"label":"uses","type":"edge","outV":7,"properties":{"skill":5}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":20,"label":"uses","type":"edge","outV":7,"properties":{"skill":4}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":17,"label":"develops","type":"edge","outV":7,"properties":{"since":2010}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":18,"label":"develops","type":"edge","outV":7,"properties":{"since":2011}}],"id":7,"label":"person","type":"vertex","properties":{"name":[{"id":1,"label":"name","value":"stephen","properties":{}}],"location":[{"id":10,"label":"location","value":"centreville","properties":{"startTime":1990,"endTime":2000}},{"id":11,"label":"location","value":"dulles","properties":{"startTime":2000,"endTime":2006}},{"id":12,"label":"location","value":"purcellville","properties":{"startTime":2006}}]}}
+{"inE":[],"outE":[{"inV":10,"inVLabel":"software","outVLabel":"person","id":22,"label":"uses","type":"edge","outV":8,"properties":{"skill":3}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":23,"label":"uses","type":"edge","outV":8,"properties":{"skill":3}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":21,"label":"develops","type":"edge","outV":8,"properties":{"since":2012}}],"id":8,"label":"person","type":"vertex","properties":{"name":[{"id":2,"label":"name","value":"matthias","properties":{}}],"location":[{"id":13,"label":"location","value":"bremen","properties":{"startTime":2004,"endTime":2007}},{"id":14,"label":"location","value":"baltimore","properties":{"startTime":2007,"endTime":2011}},{"id":15,"label":"location","value":"oakland","properties":{"startTime":2011,"endTime":2014}},{"id":16,"label":"location","value":"seattle","properties":{"startTime":2014}}]}}
+{"inE":[],"outE":[{"inV":10,"inVLabel":"software","outVLabel":"person","id":24,"label":"uses","type":"edge","outV":9,"properties":{"skill":5}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":25,"label":"uses","type":"edge","outV":9,"properties":{"skill":3}}],"id":9,"label":"person","type":"vertex","properties":{"name":[{"id":3,"label":"name","value":"daniel","properties":{}}],"location":[{"id":17,"label":"location","value":"spremberg","properties":{"startTime":1982,"endTime":2005}},{"id":18,"label":"location","value":"kaiserslautern","properties":{"startTime":2005,"endTime":2009}},{"id":19,"label":"location","value":"aachen","properties":{"startTime":2009}}]}}
+{"inE":[{"inV":10,"inVLabel":"software","outVLabel":"person","id":19,"label":"uses","type":"edge","outV":7,"properties":{"skill":5}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":22,"label":"uses","type":"edge","outV":8,"properties":{"skill":3}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":24,"label":"uses","type":"edge","outV":9,"properties":{"skill":5}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":15,"label":"uses","type":"edge","outV":1,"properties":{"skill":4}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":17,"label":"develops","type":"edge","outV":7,"properties":{"since":2010}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":21,"label":"develops","type":"edge","outV":8,"properties":{"since":2012}},{"inV":10,"inVLabel":"software","outVLabel":"person","id":13,"label":"develops","type":"edge","outV":1,"properties":{"since":2009}}],"outE":[{"inV":11,"inVLabel":"software","outVLabel":"software","id":26,"label":"traverses","
type":"edge","outV":10,"properties":{}}],"id":10,"label":"software","type":"vertex","properties":{"name":[{"id":4,"label":"name","value":"gremlin","properties":{}}]}}
+{"inE":[{"inV":11,"inVLabel":"software","outVLabel":"software","id":26,"label":"traverses","type":"edge","outV":10,"properties":{}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":16,"label":"uses","type":"edge","outV":1,"properties":{"skill":5}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":20,"label":"uses","type":"edge","outV":7,"properties":{"skill":4}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":23,"label":"uses","type":"edge","outV":8,"properties":{"skill":3}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":25,"label":"uses","type":"edge","outV":9,"properties":{"skill":3}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":18,"label":"develops","type":"edge","outV":7,"properties":{"since":2011}},{"inV":11,"inVLabel":"software","outVLabel":"person","id":14,"label":"develops","type":"edge","outV":1,"properties":{"since":2010}}],"outE":[],"id":11,"label":"software","type":"vertex","properties":{"name":[{"id":5,"label":"name","v
alue":"tinkergraph","properties":{}}]}}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/graphson/tinkerpop-modern-vertices.ldjson
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/graphson/tinkerpop-modern-vertices.ldjson b/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/graphson/tinkerpop-modern-vertices.ldjson
new file mode 100644
index 0000000..7ca4dcc
--- /dev/null
+++ b/gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/graphson/tinkerpop-modern-vertices.ldjson
@@ -0,0 +1,6 @@
+{"inE":[],"outE":[{"inV":3,"inVLabel":"software","outVLabel":"person","id":9,"label":"created","type":"edge","outV":1,"properties":{"weight":0.4}},{"inV":2,"inVLabel":"person","outVLabel":"person","id":7,"label":"knows","type":"edge","outV":1,"properties":{"weight":0.5}},{"inV":4,"inVLabel":"person","outVLabel":"person","id":8,"label":"knows","type":"edge","outV":1,"properties":{"weight":1.0}}],"id":1,"label":"person","type":"vertex","properties":{"name":[{"id":0,"label":"name","value":"marko","properties":{}}],"age":[{"id":1,"label":"age","value":29,"properties":{}}]}}
+{"inE":[{"inV":2,"inVLabel":"person","outVLabel":"person","id":7,"label":"knows","type":"edge","outV":1,"properties":{"weight":0.5}}],"outE":[],"id":2,"label":"person","type":"vertex","properties":{"name":[{"id":2,"label":"name","value":"vadas","properties":{}}],"age":[{"id":3,"label":"age","value":27,"properties":{}}]}}
+{"inE":[{"inV":3,"inVLabel":"software","outVLabel":"person","id":9,"label":"created","type":"edge","outV":1,"properties":{"weight":0.4}},{"inV":3,"inVLabel":"software","outVLabel":"person","id":11,"label":"created","type":"edge","outV":4,"properties":{"weight":0.4}},{"inV":3,"inVLabel":"software","outVLabel":"person","id":12,"label":"created","type":"edge","outV":6,"properties":{"weight":0.2}}],"outE":[],"id":3,"label":"software","type":"vertex","properties":{"name":[{"id":4,"label":"name","value":"lop","properties":{}}],"lang":[{"id":5,"label":"lang","value":"java","properties":{}}]}}
+{"inE":[{"inV":4,"inVLabel":"person","outVLabel":"person","id":8,"label":"knows","type":"edge","outV":1,"properties":{"weight":1.0}}],"outE":[{"inV":5,"inVLabel":"software","outVLabel":"person","id":10,"label":"created","type":"edge","outV":4,"properties":{"weight":1.0}},{"inV":3,"inVLabel":"software","outVLabel":"person","id":11,"label":"created","type":"edge","outV":4,"properties":{"weight":0.4}}],"id":4,"label":"person","type":"vertex","properties":{"name":[{"id":6,"label":"name","value":"josh","properties":{}}],"age":[{"id":7,"label":"age","value":32,"properties":{}}]}}
+{"inE":[{"inV":5,"inVLabel":"software","outVLabel":"person","id":10,"label":"created","type":"edge","outV":4,"properties":{"weight":1.0}}],"outE":[],"id":5,"label":"software","type":"vertex","properties":{"name":[{"id":8,"label":"name","value":"ripple","properties":{}}],"lang":[{"id":9,"label":"lang","value":"java","properties":{}}]}}
+{"inE":[],"outE":[{"inV":3,"inVLabel":"software","outVLabel":"person","id":12,"label":"created","type":"edge","outV":6,"properties":{"weight":0.2}}],"id":6,"label":"person","type":"vertex","properties":{"name":[{"id":10,"label":"name","value":"peter","properties":{}}],"age":[{"id":11,"label":"age","value":35,"properties":{}}]}}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/util/MapReduceHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/util/MapReduceHelper.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/util/MapReduceHelper.java
index 14f44c1..d6fa24b 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/util/MapReduceHelper.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/util/MapReduceHelper.java
@@ -36,6 +36,7 @@ import org.apache.tinkerpop.gremlin.hadoop.process.computer.HadoopCombine;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.HadoopMap;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.HadoopReduce;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.InputOutputHelper;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableComparator;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableIterator;
@@ -57,12 +58,13 @@ public final class MapReduceHelper {
}
public static void executeMapReduceJob(final MapReduce mapReduce, final Memory.Admin memory, final Configuration configuration) throws IOException, ClassNotFoundException, InterruptedException {
+ final boolean vertexProgramExists = configuration.get(VertexProgram.VERTEX_PROGRAM, null) != null;
final Configuration newConfiguration = new Configuration(configuration);
final BaseConfiguration apacheConfiguration = new BaseConfiguration();
apacheConfiguration.setDelimiterParsingDisabled(true);
mapReduce.storeState(apacheConfiguration);
ConfUtil.mergeApacheIntoHadoopConfiguration(apacheConfiguration, newConfiguration);
- if (!mapReduce.doStage(MapReduce.Stage.MAP)) {
+ if (!mapReduce.doStage(MapReduce.Stage.MAP)) { // TODO : how is this possible?
final Path memoryPath = new Path(configuration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + mapReduce.getMemoryKey());
if (newConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, SequenceFileOutputFormat.class, OutputFormat.class).equals(SequenceFileOutputFormat.class))
mapReduce.addResultToMemory(memory, new ObjectWritableIterator(configuration, memoryPath));
@@ -94,10 +96,12 @@ public final class MapReduceHelper {
job.setMapOutputValueClass(ObjectWritable.class);
job.setOutputKeyClass(ObjectWritable.class);
job.setOutputValueClass(ObjectWritable.class);
- job.setInputFormatClass((Class) newConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class));
+ job.setInputFormatClass(vertexProgramExists ?
+ InputOutputHelper.getInputFormat((Class) newConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, OutputFormat.class)) :
+ (Class) newConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class));
job.setOutputFormatClass(newConfiguration.getClass(Constants.GREMLIN_HADOOP_MEMORY_OUTPUT_FORMAT, SequenceFileOutputFormat.class, OutputFormat.class)); // TODO: Make this configurable
// if there is no vertex program, then grab the graph from the input location
- final Path graphPath = configuration.get(VertexProgram.VERTEX_PROGRAM, null) != null ?
+ final Path graphPath = vertexProgramExists ?
new Path(newConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.SYSTEM_G) :
new Path(newConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
Path memoryPath = new Path(newConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + (reduceSort.isPresent() ? mapReduce.getMemoryKey() + "-temp" : mapReduce.getMemoryKey()));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/InputOutputHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/InputOutputHelper.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/InputOutputHelper.java
new file mode 100644
index 0000000..e08d90e
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/InputOutputHelper.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.structure.io;
+
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.script.ScriptInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.script.ScriptOutputFormat;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class InputOutputHelper {
+
+ private InputOutputHelper() {
+
+ }
+
+ public static Class<? extends InputFormat> getInputFormat(final Class<? extends OutputFormat> outputFormat) {
+ if (outputFormat.equals(GryoOutputFormat.class))
+ return GryoInputFormat.class;
+ else if (outputFormat.equals(GraphSONOutputFormat.class))
+ return GraphSONInputFormat.class;
+ else if (outputFormat.equals(ScriptOutputFormat.class))
+ return ScriptInputFormat.class;
+ else
+ throw new IllegalArgumentException("The provided output format does not have a known input format: " + outputFormat.getCanonicalName());
+ }
+
+ public static Class<? extends OutputFormat> getOutputFormat(final Class<? extends InputFormat> inputFormat) {
+ if (inputFormat.equals(GryoInputFormat.class))
+ return GryoOutputFormat.class;
+ else if (inputFormat.equals(GraphSONInputFormat.class))
+ return GraphSONOutputFormat.class;
+ else if (inputFormat.equals(ScriptInputFormat.class))
+ return ScriptOutputFormat.class;
+ else
+ throw new IllegalArgumentException("The provided input format does not have a known output format: " + inputFormat.getCanonicalName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONInputFormat.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONInputFormat.java
index c33db48..d91586e 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONInputFormat.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONInputFormat.java
@@ -18,7 +18,6 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -29,6 +28,7 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONOutputFormat.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONOutputFormat.java
index bdb2281..1acb320 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONOutputFormat.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONOutputFormat.java
@@ -18,11 +18,11 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.CommonFileOutputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.CommonFileOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -40,4 +40,5 @@ public class GraphSONOutputFormat extends CommonFileOutputFormat {
public RecordWriter<NullWritable, VertexWritable> getRecordWriter(final TaskAttemptContext job, final DataOutputStream outputStream) throws IOException, InterruptedException {
return new GraphSONRecordWriter(outputStream);
}
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoOutputFormat.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoOutputFormat.java
index 7a214f8..1327d90 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoOutputFormat.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoOutputFormat.java
@@ -18,11 +18,11 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.CommonFileOutputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.CommonFileOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import java.io.DataOutputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptInputFormat.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptInputFormat.java
index 3dbbef1..73ade5f 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptInputFormat.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptInputFormat.java
@@ -18,7 +18,6 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.structure.io.script;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -29,6 +28,7 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptOutputFormat.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptOutputFormat.java
index 691a11f..f1bec62 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptOutputFormat.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptOutputFormat.java
@@ -18,11 +18,11 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.structure.io.script;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.CommonFileOutputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.CommonFileOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import java.io.DataOutputStream;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReader.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReader.java
index 5e0c9e0..1c322a7 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReader.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReader.java
@@ -30,6 +30,7 @@ import org.apache.tinkerpop.gremlin.groovy.DefaultImportCustomizerProvider;
import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.process.T;
+import org.apache.tinkerpop.gremlin.process.computer.util.ScriptEngineCache;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
@@ -63,7 +64,7 @@ public class ScriptRecordReader extends RecordReader<NullWritable, VertexWritabl
this.lineRecordReader.initialize(genericSplit, context);
final Configuration configuration = context.getConfiguration();
this.engine = new GremlinGroovyScriptEngine(new DefaultImportCustomizerProvider(), null, Long.MAX_VALUE);
- // this.engine = ScriptEngineCache.get(configuration.get(SCRIPT_ENGINE, ScriptEngineCache.DEFAULT_SCRIPT_ENGINE));
+ //this.engine = ScriptEngineCache.get(configuration.get(SCRIPT_ENGINE, ScriptEngineCache.DEFAULT_SCRIPT_ENGINE));
final FileSystem fs = FileSystem.get(configuration);
try {
this.engine.eval(new InputStreamReader(fs.open(new Path(configuration.get(SCRIPT_FILE)))));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
index c3aed34..e09353b 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
@@ -33,6 +33,8 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraphVariables;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -77,7 +79,10 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
}
final List<String> graphsonResources = Arrays.asList(
- "grateful-dead-vertices.ldjson");
+ "tinkerpop-modern-vertices.ldjson",
+ "grateful-dead-vertices.ldjson",
+ "tinkerpop-classic-vertices.ldjson",
+ "tinkerpop-crew-vertices.ldjson");
for (final String fileName : graphsonResources) {
PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
}
@@ -98,7 +103,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName) {
return new HashMap<String, Object>() {{
put(Graph.GRAPH, HadoopGraph.class.getName());
- put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GraphSONInputFormat.class.getCanonicalName());
put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
put(Constants.GREMLIN_HADOOP_MEMORY_OUTPUT_FORMAT, SequenceFileOutputFormat.class.getCanonicalName());
put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
@@ -137,13 +142,13 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead-vertices.kryo"));
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead-vertices.ldjson"));
} else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern-vertices.kryo"));
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern-vertices.ldjson"));
} else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic-vertices.kryo"));
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic-vertices.ldjson"));
} else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew-vertices.kryo"));
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew-vertices.ldjson"));
} else {
throw new RuntimeException("Could not load graph with " + graphData);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be80e558/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
index 86572a2..e455bda 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
@@ -24,7 +24,6 @@ import org.apache.tinkerpop.gremlin.process.T;
import org.apache.tinkerpop.gremlin.process.Traversal;
import org.apache.tinkerpop.gremlin.process.graph.traversal.GraphTraversalContext;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategy;
-import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ReadOnlyStrategy;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -148,7 +147,7 @@ public class TinkerGraphTest {
public void testPlay3() throws Exception {
Graph graph = TinkerFactory.createModern();
GraphTraversalContext g = graph.traversal(GraphTraversalContext.build().strategy(new ElementIdStrategy("name")));
- Traversal t = g.V("marko").out("knows").in("knows").has(T.id,"marko");
+ Traversal t = g.V("marko").out("knows").in("knows").has(T.id, "marko");
System.out.println(t);
t.forEachRemaining(System.out::println);
System.out.println(t);
@@ -277,6 +276,16 @@ public class TinkerGraphTest {
* No assertions. Just write out the graph for convenience.
*/
@Test
+ public void shouldWriteCrewVerticesAsGraphSON() throws IOException {
+ final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-crew-vertices.ldjson");
+ GraphSONWriter.build().create().writeVertices(os, TinkerFactory.createTheCrew().traversal().V(), Direction.BOTH);
+ os.close();
+ }
+
+ /**
+ * No assertions. Just write out the graph for convenience.
+ */
+ @Test
public void shouldWriteCrewVerticesAsGryo() throws IOException {
final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-crew-vertices.kryo");
GryoWriter.build().create().writeVertices(os, TinkerFactory.createTheCrew().traversal().V(), Direction.BOTH);