You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/03 19:10:34 UTC
[2/8] incubator-tinkerpop git commit: Renamed Gremlin Kryo to simply
Gryo.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/929a2889/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReaderWriterTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReaderWriterTest.java
new file mode 100644
index 0000000..bf03e68
--- /dev/null
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReaderWriterTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo;
+
+import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Joshua Shinavier (http://fortytwo.net)
+ */
+public class GryoRecordReaderWriterTest {
+ @Test
+ public void testAll() throws Exception {
+ Configuration conf = new Configuration(false);
+ conf.set("fs.file.impl", LocalFileSystem.class.getName());
+ conf.set("fs.default.name", "file:///");
+
+ File testFile = new File(HadoopGraphProvider.PATHS.get("grateful-dead-vertices.kryo"));
+ FileSplit split = new FileSplit(
+ new Path(testFile.getAbsoluteFile().toURI().toString()), 0,
+ testFile.length(), null);
+ System.out.println("reading Gryo file " + testFile.getAbsolutePath() + " (" + testFile.length() + " bytes)");
+
+ GryoInputFormat inputFormat = ReflectionUtils.newInstance(GryoInputFormat.class, conf);
+ TaskAttemptContext job = new TaskAttemptContext(conf, new TaskAttemptID());
+ RecordReader reader = inputFormat.createRecordReader(split, job);
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ try (DataOutputStream dos = new DataOutputStream(bos)) {
+ GryoOutputFormat outputFormat = new GryoOutputFormat();
+ RecordWriter writer = outputFormat.getRecordWriter(job, dos);
+
+ float lastProgress = -1f;
+ int count = 0;
+ boolean foundKeyValue = false;
+ while (reader.nextKeyValue()) {
+ //System.out.println("" + reader.getProgress() + "> " + reader.getCurrentKey() + ": " + reader.getCurrentValue());
+ count++;
+ float progress = reader.getProgress();
+ assertTrue(progress >= lastProgress);
+ assertEquals(NullWritable.class, reader.getCurrentKey().getClass());
+ VertexWritable v = (VertexWritable) reader.getCurrentValue();
+ writer.write(NullWritable.get(), v);
+
+ Vertex vertex = v.get();
+ assertEquals(Integer.class, vertex.id().getClass());
+
+ Object value = vertex.property("name");
+ if (null != value && ((Property) value).value().equals("SUGAR MAGNOLIA")) {
+ foundKeyValue = true;
+ assertEquals(92, count(vertex.outE().toList()));
+ assertEquals(77, count(vertex.inE().toList()));
+ }
+
+ lastProgress = progress;
+ }
+ assertEquals(808, count);
+ assertTrue(foundKeyValue);
+ }
+
+ //System.out.println("bos: " + new String(bos.toByteArray()));
+ String[] lines = new String(bos.toByteArray()).split("\\x3a\\x15.\\x11\\x70...");
+ assertEquals(808, lines.length);
+ String line42 = lines[41];
+ //System.out.println("line42: " + line42);
+ assertTrue(line42.contains("ITS ALL OVER NO"));
+ }
+
+ private <T> long count(final Iterable<T> iter) {
+ long count = 0;
+ for (T anIter : iter) {
+ count++;
+ }
+
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/929a2889/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/kryo/KryoRecordReaderWriterTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/kryo/KryoRecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/kryo/KryoRecordReaderWriterTest.java
deleted file mode 100644
index ed77145..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/kryo/KryoRecordReaderWriterTest.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.structure.io.kryo;
-
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.structure.Property;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author Joshua Shinavier (http://fortytwo.net)
- */
-public class KryoRecordReaderWriterTest {
- @Test
- public void testAll() throws Exception {
- Configuration conf = new Configuration(false);
- conf.set("fs.file.impl", LocalFileSystem.class.getName());
- conf.set("fs.default.name", "file:///");
-
- File testFile = new File(HadoopGraphProvider.PATHS.get("grateful-dead-vertices.gio"));
- FileSplit split = new FileSplit(
- new Path(testFile.getAbsoluteFile().toURI().toString()), 0,
- testFile.length(), null);
- System.out.println("reading Gremlin Kryo file " + testFile.getAbsolutePath() + " (" + testFile.length() + " bytes)");
-
- KryoInputFormat inputFormat = ReflectionUtils.newInstance(KryoInputFormat.class, conf);
- TaskAttemptContext job = new TaskAttemptContext(conf, new TaskAttemptID());
- RecordReader reader = inputFormat.createRecordReader(split, job);
-
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- try (DataOutputStream dos = new DataOutputStream(bos)) {
- KryoOutputFormat outputFormat = new KryoOutputFormat();
- RecordWriter writer = outputFormat.getRecordWriter(job, dos);
-
- float lastProgress = -1f;
- int count = 0;
- boolean foundKeyValue = false;
- while (reader.nextKeyValue()) {
- //System.out.println("" + reader.getProgress() + "> " + reader.getCurrentKey() + ": " + reader.getCurrentValue());
- count++;
- float progress = reader.getProgress();
- assertTrue(progress >= lastProgress);
- assertEquals(NullWritable.class, reader.getCurrentKey().getClass());
- VertexWritable v = (VertexWritable) reader.getCurrentValue();
- writer.write(NullWritable.get(), v);
-
- Vertex vertex = v.get();
- assertEquals(Integer.class, vertex.id().getClass());
-
- Object value = vertex.property("name");
- if (null != value && ((Property) value).value().equals("SUGAR MAGNOLIA")) {
- foundKeyValue = true;
- assertEquals(92, count(vertex.outE().toList()));
- assertEquals(77, count(vertex.inE().toList()));
- }
-
- lastProgress = progress;
- }
- assertEquals(808, count);
- assertTrue(foundKeyValue);
- }
-
- //System.out.println("bos: " + new String(bos.toByteArray()));
- String[] lines = new String(bos.toByteArray()).split("\\x3a\\x15.\\x11\\x70...");
- assertEquals(808, lines.length);
- String line42 = lines[41];
- //System.out.println("line42: " + line42);
- assertTrue(line42.contains("ITS ALL OVER NO"));
- }
-
- private <T> long count(final Iterable<T> iter) {
- long count = 0;
- for (T anIter : iter) {
- count++;
- }
-
- return count;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/929a2889/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 30666dd..8acac72 100644
--- a/pom.xml
+++ b/pom.xml
@@ -216,7 +216,7 @@ limitations under the License.
<exclude>docs/static/**</exclude>
<exclude>docs/stylesheets/**</exclude>
<exclude>**/target/**</exclude>
- <exclude>**/*.gio</exclude>
+ <exclude>**/*.kryo</exclude>
<exclude>**/*.iml</exclude>
<exclude>**/*.json</exclude>
<exclude>**/*.xml</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/929a2889/tinkergraph-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/pom.xml b/tinkergraph-gremlin/pom.xml
index 05c7214..7361bd3 100644
--- a/tinkergraph-gremlin/pom.xml
+++ b/tinkergraph-gremlin/pom.xml
@@ -128,19 +128,19 @@ limitations under the License.
</goals>
<configuration>
<outputDirectory>
- ../gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/kryo
+ ../gremlin-test/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/gryo
</outputDirectory>
<resources>
<resource>
<directory>${io.tmp.dir}</directory>
<filtering>false</filtering>
<includes>
- <include>**/*.gio</include>
+ <include>**/*.kryo</include>
</includes>
</resource>
</resources>
<nonFilteredFileExtensions>
- <nonFilteredFileExtension>gio</nonFilteredFileExtension>
+ <nonFilteredFileExtension>kryo</nonFilteredFileExtension>
</nonFilteredFileExtensions>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/929a2889/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
index 2ce857d..fdea748 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
@@ -29,8 +29,8 @@ import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
-import org.apache.tinkerpop.gremlin.structure.io.kryo.KryoReader;
-import org.apache.tinkerpop.gremlin.structure.io.kryo.KryoWriter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter;
import org.apache.tinkerpop.gremlin.util.StreamFactory;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -39,7 +39,6 @@ import org.junit.Test;
import java.io.*;
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
@@ -198,9 +197,9 @@ public class TinkerGraphTest {
* No assertions. Just write out the graph for convenience.
*/
@Test
- public void shouldWriteClassicGraphAsKryo() throws IOException {
- final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-classic.gio");
- KryoWriter.build().create().writeGraph(os, TinkerFactory.createClassic());
+ public void shouldWriteClassicGraphAsGryo() throws IOException {
+ final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-classic.kryo");
+ GryoWriter.build().create().writeGraph(os, TinkerFactory.createClassic());
os.close();
}
@@ -208,9 +207,9 @@ public class TinkerGraphTest {
* No assertions. Just write out the graph for convenience.
*/
@Test
- public void shouldWriteModernGraphAsKryo() throws IOException {
- final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-modern.gio");
- KryoWriter.build().create().writeGraph(os, TinkerFactory.createModern());
+ public void shouldWriteModernGraphAsGryo() throws IOException {
+ final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-modern.kryo");
+ GryoWriter.build().create().writeGraph(os, TinkerFactory.createModern());
os.close();
}
@@ -218,9 +217,9 @@ public class TinkerGraphTest {
* No assertions. Just write out the graph for convenience.
*/
@Test
- public void shouldWriteCrewGraphAsKryo() throws IOException {
- final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-crew.gio");
- KryoWriter.build().create().writeGraph(os, TinkerFactory.createTheCrew());
+ public void shouldWriteCrewGraphAsGryo() throws IOException {
+ final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-crew.kryo");
+ GryoWriter.build().create().writeGraph(os, TinkerFactory.createTheCrew());
os.close();
}
@@ -228,9 +227,9 @@ public class TinkerGraphTest {
* No assertions. Just write out the graph for convenience.
*/
@Test
- public void shouldWriteClassicVerticesAsKryo() throws IOException {
- final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-classic-vertices.gio");
- KryoWriter.build().create().writeVertices(os, TinkerFactory.createClassic().V(), Direction.BOTH);
+ public void shouldWriteClassicVerticesAsGryo() throws IOException {
+ final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-classic-vertices.kryo");
+ GryoWriter.build().create().writeVertices(os, TinkerFactory.createClassic().V(), Direction.BOTH);
os.close();
}
@@ -248,9 +247,9 @@ public class TinkerGraphTest {
* No assertions. Just write out the graph for convenience.
*/
@Test
- public void shouldWriteModernVerticesAsKryo() throws IOException {
- final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-modern-vertices.gio");
- KryoWriter.build().create().writeVertices(os, TinkerFactory.createModern().V(), Direction.BOTH);
+ public void shouldWriteModernVerticesAsGryo() throws IOException {
+ final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-modern-vertices.kryo");
+ GryoWriter.build().create().writeVertices(os, TinkerFactory.createModern().V(), Direction.BOTH);
os.close();
}
@@ -268,9 +267,9 @@ public class TinkerGraphTest {
* No assertions. Just write out the graph for convenience.
*/
@Test
- public void shouldWriteCrewVerticesAsKryo() throws IOException {
- final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-crew-vertices.gio");
- KryoWriter.build().create().writeVertices(os, TinkerFactory.createTheCrew().V(), Direction.BOTH);
+ public void shouldWriteCrewVerticesAsGryo() throws IOException {
+ final OutputStream os = new FileOutputStream(tempPath + "tinkerpop-crew-vertices.kryo");
+ GryoWriter.build().create().writeVertices(os, TinkerFactory.createTheCrew().V(), Direction.BOTH);
os.close();
}
@@ -612,12 +611,12 @@ public class TinkerGraphTest {
@Test
public void shouldWriteGratefulDead() throws IOException {
final Graph g = TinkerGraph.open();
- final GraphReader reader = KryoReader.build().create();
- try (final InputStream stream = AbstractGremlinTest.class.getResourceAsStream("/org/apache/tinkerpop/gremlin/structure/io/kryo/grateful-dead.gio")) {
+ final GraphReader reader = GryoReader.build().create();
+ try (final InputStream stream = AbstractGremlinTest.class.getResourceAsStream("/org/apache/tinkerpop/gremlin/structure/io/gryo/grateful-dead.kryo")) {
reader.readGraph(stream, g);
}
- /* keep this hanging around because changes to kryo format will need grateful dead generated from json so you can generate the gio
+ /* keep this hanging around because changes to gryo format will need grateful dead generated from json so you can generate the gio
final GraphReader reader = GraphSONReader.build().embedTypes(true).create();
try (final InputStream stream = AbstractGremlinTest.class.getResourceAsStream("/org/apache/tinkerpop/gremlin/structure/io/graphson/grateful-dead.json")) {
reader.readGraph(stream, g);
@@ -651,8 +650,8 @@ public class TinkerGraphTest {
}).iterate();
- final OutputStream os = new FileOutputStream(tempPath + "grateful-dead.gio");
- KryoWriter.build().create().writeGraph(os, ng);
+ final OutputStream os = new FileOutputStream(tempPath + "grateful-dead.kryo");
+ GryoWriter.build().create().writeGraph(os, ng);
os.close();
final OutputStream os2 = new FileOutputStream(tempPath + "grateful-dead.json");
@@ -663,8 +662,8 @@ public class TinkerGraphTest {
GraphMLWriter.build().create().writeGraph(os3, g);
os3.close();
- final OutputStream os4 = new FileOutputStream(tempPath + "grateful-dead-vertices.gio");
- KryoWriter.build().create().writeVertices(os4, g.V(), Direction.BOTH);
+ final OutputStream os4 = new FileOutputStream(tempPath + "grateful-dead-vertices.kryo");
+ GryoWriter.build().create().writeVertices(os4, g.V(), Direction.BOTH);
os.close();
final OutputStream os5 = new FileOutputStream(tempPath + "grateful-dead-vertices.ldjson");