You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2017/07/10 21:26:25 UTC
[43/50] [abbrv] tinkerpop git commit: GraphSONRecordReader/Writer can
now be configured to use either GraphSON V2 or V3. Added a test case to
verfify proper behavior of parser from GraphSON V3 on HDFS.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1225bd09/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
index 3ff8e2a..1e378f2 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
@@ -40,6 +40,7 @@ public final class Constants {
public static final String GREMLIN_HADOOP_GRAPH_FILTER = "gremlin.hadoop.graphFilter";
public static final String GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER = "gremlin.hadoop.defaultGraphComputer";
public static final String GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR = "gremlin.hadoop.vertexProgramInterceptor";
+ public static final String GREMLIN_HADOOP_GRAPHSON_VERSION = "gremlin.hadoop.graphSONVersion";
public static final String GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE = "gremlin.hadoop.jarsInDistributedCache";
public static final String HIDDEN_G = Graph.Hidden.hide("g");
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1225bd09/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordReader.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordReader.java
index 07bd303..1b8cb3c 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordReader.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordReader.java
@@ -58,7 +58,7 @@ public final class GraphSONRecordReader extends RecordReader<NullWritable, Verte
this.hasEdges = context.getConfiguration().getBoolean(Constants.GREMLIN_HADOOP_GRAPH_READER_HAS_EDGES, true);
this.graphsonReader = GraphSONReader.build().mapper(
GraphSONMapper.build().
- version(GraphSONVersion.V2_0).
+ version(GraphSONVersion.valueOf(context.getConfiguration().get(Constants.GREMLIN_HADOOP_GRAPHSON_VERSION, "V3_0"))).
typeInfo(TypeInfo.PARTIAL_TYPES).
addRegistries(IoRegistryHelper.createRegistries(ConfUtil.makeApacheConfiguration(context.getConfiguration()))).create()).create();
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1225bd09/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordWriter.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordWriter.java
index 5ea058f..a5687de 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordWriter.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordWriter.java
@@ -60,7 +60,7 @@ public final class GraphSONRecordWriter extends RecordWriter<NullWritable, Verte
this.hasEdges = configuration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_WRITER_HAS_EDGES, true);
this.graphsonWriter = GraphSONWriter.build().mapper(
GraphSONMapper.build().
- version(GraphSONVersion.V2_0).
+ version(GraphSONVersion.valueOf(configuration.get(Constants.GREMLIN_HADOOP_GRAPHSON_VERSION, "V3_0"))).
typeInfo(TypeInfo.PARTIAL_TYPES).
addRegistries(IoRegistryHelper.createRegistries(ConfUtil.makeApacheConfiguration(configuration))).create()).create();
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1225bd09/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
index 2c51524..346b731 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
@@ -87,6 +87,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
final List<String> graphsonResources = Arrays.asList(
"tinkerpop-modern-v2d0-typed.json",
"grateful-dead-v2d0-typed.json",
+ "grateful-dead-v3d0-typed.json",
"tinkerpop-classic-v2d0-typed.json",
"tinkerpop-crew-v2d0-typed.json");
for (final String fileName : graphsonResources) {
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1225bd09/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java
index f3c079b..d2d2316 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java
@@ -56,6 +56,9 @@ import static org.junit.Assert.assertTrue;
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
public abstract class RecordReaderWriterTest {
+ // extra configurations that a extending class can add prior to test execution
+ protected Configuration configuration = new Configuration();
+
private static final Logger logger = LoggerFactory.getLogger(RecordReaderWriterTest.class);
protected abstract String getInputFilename();
@@ -74,6 +77,7 @@ public abstract class RecordReaderWriterTest {
final Class<? extends OutputFormat<NullWritable, VertexWritable>> outputFormatClass = getOutputFormat();
final File outputDirectory = TestHelper.makeTestDataPath(inputFormatClass, "hadoop-record-reader-writer-test");
final Configuration config = configure(outputDirectory);
+ config.addResource(this.configuration);
validateFileSplits(splits, config, inputFormatClass, Optional.of(outputFormatClass));
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1225bd09/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordReaderWriterTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordReaderWriterTest.java
deleted file mode 100644
index 903bef5..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONRecordReaderWriterTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson;
-
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.RecordReaderWriterTest;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- * @author Daniel Kuppitz (http://gremlin.guru)
- */
-public class GraphSONRecordReaderWriterTest extends RecordReaderWriterTest {
-
- @Override
- protected String getInputFilename() {
- return "grateful-dead-v2d0-typed.json";
- }
-
- @Override
- protected Class<? extends InputFormat<NullWritable, VertexWritable>> getInputFormat() {
- return GraphSONInputFormat.class;
- }
-
- @Override
- protected Class<? extends OutputFormat<NullWritable, VertexWritable>> getOutputFormat() {
- return GraphSONOutputFormat.class;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1225bd09/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONV2d0RecordReaderWriterTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONV2d0RecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONV2d0RecordReaderWriterTest.java
new file mode 100644
index 0000000..eca34a5
--- /dev/null
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONV2d0RecordReaderWriterTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.RecordReaderWriterTest;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ * @author Daniel Kuppitz (http://gremlin.guru)
+ */
+public class GraphSONV2d0RecordReaderWriterTest extends RecordReaderWriterTest {
+
+ public GraphSONV2d0RecordReaderWriterTest() {
+ super.configuration.set(Constants.GREMLIN_HADOOP_GRAPHSON_VERSION, GraphSONVersion.V2_0.name());
+ }
+
+ @Override
+ protected String getInputFilename() {
+ return "grateful-dead-v2d0-typed.json";
+ }
+
+ @Override
+ protected Class<? extends InputFormat<NullWritable, VertexWritable>> getInputFormat() {
+ return GraphSONInputFormat.class;
+ }
+
+ @Override
+ protected Class<? extends OutputFormat<NullWritable, VertexWritable>> getOutputFormat() {
+ return GraphSONOutputFormat.class;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1225bd09/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONV3d0RecordReaderWriterTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONV3d0RecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONV3d0RecordReaderWriterTest.java
new file mode 100644
index 0000000..1783fd9
--- /dev/null
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/graphson/GraphSONV3d0RecordReaderWriterTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.RecordReaderWriterTest;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class GraphSONV3d0RecordReaderWriterTest extends RecordReaderWriterTest {
+
+ public GraphSONV3d0RecordReaderWriterTest() {
+ // should be default
+ // super.configuration.set(Constants.GREMLIN_HADOOP_GRAPHSON_VERSION, GraphSONVersion.V3_0.name());
+ }
+
+ @Override
+ protected String getInputFilename() {
+ return "grateful-dead-v3d0-typed.json";
+ }
+
+ @Override
+ protected Class<? extends InputFormat<NullWritable, VertexWritable>> getInputFormat() {
+ return GraphSONInputFormat.class;
+ }
+
+ @Override
+ protected Class<? extends OutputFormat<NullWritable, VertexWritable>> getOutputFormat() {
+ return GraphSONOutputFormat.class;
+ }
+}
+