You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2018/07/31 12:17:23 UTC
[06/38] tinkerpop git commit: TINKERPOP-1996 Have the basics of OLAP
read()/write() steps working
TINKERPOP-1996 Have the basics of OLAP read()/write() steps working
This is still fairly skeletal at this point. Just trying to make sure things work properly before building read()/write() out fully.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/0785090f
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/0785090f
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/0785090f
Branch: refs/heads/master
Commit: 0785090f67ff88a834f77c7181a69594049c65ea
Parents: ec1d05f
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Jul 10 12:21:52 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:39:29 2018 -0400
----------------------------------------------------------------------
.../decoration/VertexProgramStrategy.java | 34 ++++----
.../process/traversal/TraversalStrategies.java | 2 +
.../gremlin/process/traversal/step/Reading.java | 27 +++++++
.../gremlin/process/traversal/step/Writing.java | 26 +++++++
.../process/traversal/step/map/ReadStep.java | 10 ++-
.../process/traversal/step/map/WriteStep.java | 8 +-
.../strategy/verification/IoUsageStrategy.java | 8 ++
.../decoration/VertexProgramStrategyTest.java | 3 +
.../gremlin/process/ProcessStandardSuite.java | 2 +
.../process/traversal/step/map/ReadTest.java | 64 +++++++++++++++
.../traversal/step/map/HadoopReadStep.java | 82 ++++++++++++++++++++
.../traversal/step/map/HadoopWriteStep.java | 82 ++++++++++++++++++++
.../traversal/strategy/HadoopIoStrategy.java | 68 ++++++++++++++++
.../gremlin/hadoop/structure/HadoopGraph.java | 8 ++
14 files changed, 407 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java
index 89e40cb..c83039a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java
@@ -34,6 +34,8 @@ import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Reading;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Writing;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
@@ -100,23 +102,26 @@ public final class VertexProgramStrategy extends AbstractTraversalStrategy<Trave
// wrap all non-VertexComputing steps into a TraversalVertexProgramStep
currentStep = traversal.getStartStep();
- while (!(currentStep instanceof EmptyStep)) {
- Traversal.Admin<?, ?> computerTraversal = new DefaultTraversal<>();
- Step<?, ?> firstLegalOLAPStep = getFirstLegalOLAPStep(currentStep);
- Step<?, ?> lastLegalOLAPStep = getLastLegalOLAPStep(currentStep);
- if (!(firstLegalOLAPStep instanceof EmptyStep)) {
- int index = TraversalHelper.stepIndex(firstLegalOLAPStep, traversal);
- TraversalHelper.removeToTraversal(firstLegalOLAPStep, lastLegalOLAPStep.getNextStep(), (Traversal.Admin) computerTraversal);
- final TraversalVertexProgramStep traversalVertexProgramStep = new TraversalVertexProgramStep(traversal, computerTraversal);
- traversal.addStep(index, traversalVertexProgramStep);
- }
- currentStep = traversal.getStartStep();
+ if (!(currentStep instanceof Reading) && !(currentStep instanceof Writing)) {
while (!(currentStep instanceof EmptyStep)) {
- if (!(currentStep instanceof VertexComputing))
- break;
- currentStep = currentStep.getNextStep();
+ final Traversal.Admin<?, ?> computerTraversal = new DefaultTraversal<>();
+ final Step<?, ?> firstLegalOLAPStep = getFirstLegalOLAPStep(currentStep);
+ final Step<?, ?> lastLegalOLAPStep = getLastLegalOLAPStep(currentStep);
+ if (!(firstLegalOLAPStep instanceof EmptyStep)) {
+ final int index = TraversalHelper.stepIndex(firstLegalOLAPStep, traversal);
+ TraversalHelper.removeToTraversal(firstLegalOLAPStep, lastLegalOLAPStep.getNextStep(), (Traversal.Admin) computerTraversal);
+ final TraversalVertexProgramStep traversalVertexProgramStep = new TraversalVertexProgramStep(traversal, computerTraversal);
+ traversal.addStep(index, traversalVertexProgramStep);
+ }
+ currentStep = traversal.getStartStep();
+ while (!(currentStep instanceof EmptyStep)) {
+ if (!(currentStep instanceof VertexComputing))
+ break;
+ currentStep = currentStep.getNextStep();
+ }
}
}
+
// if the last vertex computing step is a TraversalVertexProgramStep convert to OLTP with ComputerResultStep
TraversalHelper.getLastStepOfAssignableClass(VertexComputing.class, traversal).ifPresent(step -> {
if (step instanceof TraversalVertexProgramStep) {
@@ -126,6 +131,7 @@ public final class VertexProgramStrategy extends AbstractTraversalStrategy<Trave
TraversalHelper.insertAfterStep(computerResultStep, (Step) step, traversal);
}
});
+
// if there is a dangling vertex computing step, add an identity traversal (solve this in the future with a specialized MapReduce)
if (traversal.getEndStep() instanceof VertexComputing && !(traversal.getEndStep() instanceof TraversalVertexProgramStep)) {
final TraversalVertexProgramStep traversalVertexProgramStep = new TraversalVertexProgramStep(traversal, __.identity().asAdmin());
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalStrategies.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalStrategies.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalStrategies.java
index ef3e841..66b0236 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalStrategies.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalStrategies.java
@@ -36,6 +36,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.Path
import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.CountStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.RepeatUnrollStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.IoUsageStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.StandardVerificationStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversalStrategies;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -222,6 +223,7 @@ public interface TraversalStrategies extends Serializable, Cloneable {
PathRetractionStrategy.instance(),
LazyBarrierStrategy.instance(),
ProfileStrategy.instance(),
+ IoUsageStrategy.instance(),
StandardVerificationStrategy.instance());
GRAPH_CACHE.put(Graph.class, graphStrategies);
GRAPH_CACHE.put(EmptyGraph.class, new DefaultTraversalStrategies());
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Reading.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Reading.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Reading.java
new file mode 100644
index 0000000..b79378b
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Reading.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.traversal.step;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public interface Reading extends Configuring {
+
+ public String getLocalFile();
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Writing.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Writing.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Writing.java
new file mode 100644
index 0000000..7b455f2
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Writing.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.traversal.step;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public interface Writing extends Configuring {
+ public String getLocalFile();
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadStep.java
index afc3e53..2612d18 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadStep.java
@@ -22,6 +22,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.TraverserGenerator;
import org.apache.tinkerpop.gremlin.process.traversal.step.Configuring;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Reading;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException;
@@ -43,7 +44,7 @@ import java.util.Map;
*
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
-public class ReadStep extends AbstractStep<Map<String,Object>, Map<String,Object>> implements Configuring {
+public class ReadStep extends AbstractStep<Map<String,Object>, Map<String,Object>> implements Reading {
private Parameters parameters = new Parameters();
private boolean first = true;
@@ -54,12 +55,16 @@ public class ReadStep extends AbstractStep<Map<String,Object>, Map<String,Object
if (null == localFile || localFile.isEmpty())
throw new IllegalArgumentException("localFile cannot be null or empty");
- if (!new File(localFile).exists()) throw new IllegalStateException(localFile + " does not exist");
this.localFile = localFile;
}
@Override
+ public String getLocalFile() {
+ return localFile;
+ }
+
+ @Override
public Parameters getParameters() {
return this.parameters;
}
@@ -76,6 +81,7 @@ public class ReadStep extends AbstractStep<Map<String,Object>, Map<String,Object
this.first = false;
final TraverserGenerator generator = this.getTraversal().getTraverserGenerator();
final File file = new File(localFile);
+ if (!file.exists()) throw new IllegalStateException(localFile + " does not exist");
try (final InputStream stream = new FileInputStream(file)) {
final Graph graph = (Graph) this.traversal.getGraph().get();
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteStep.java
index e9346cf..3a035e7 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteStep.java
@@ -22,6 +22,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.TraverserGenerator;
import org.apache.tinkerpop.gremlin.process.traversal.step.Configuring;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Writing;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException;
@@ -43,7 +44,7 @@ import java.util.Map;
*
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
-public class WriteStep extends AbstractStep<Map<String,Object>, Map<String,Object>> implements Configuring {
+public class WriteStep extends AbstractStep<Map<String,Object>, Map<String,Object>> implements Writing {
private Parameters parameters = new Parameters();
private boolean first = true;
@@ -59,6 +60,11 @@ public class WriteStep extends AbstractStep<Map<String,Object>, Map<String,Objec
}
@Override
+ public String getLocalFile() {
+ return localFile;
+ }
+
+ @Override
public Parameters getParameters() {
return this.parameters;
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/IoUsageStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/IoUsageStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/IoUsageStrategy.java
index 95761ff..e9c7ec8 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/IoUsageStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/IoUsageStrategy.java
@@ -26,6 +26,9 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteStep;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
+import java.util.Collections;
+import java.util.Set;
+
/**
* {@code IoUsageStrategy} prevents the {@link GraphTraversalSource#read(String)} and
* {@link GraphTraversalSource#write(String)} steps from being used outside of their intended scope, which is as the
@@ -57,4 +60,9 @@ public final class IoUsageStrategy extends AbstractTraversalStrategy<TraversalSt
public static IoUsageStrategy instance() {
return INSTANCE;
}
+
+ @Override
+ public Set<Class<? extends VerificationStrategy>> applyPrior() {
+ return Collections.singleton(ComputerVerificationStrategy.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java
index c091bb3..71cb22d 100644
--- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java
@@ -32,6 +32,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversalStrategies;
import org.apache.tinkerpop.gremlin.process.traversal.util.EmptyTraversal;
+import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -68,6 +69,8 @@ public class VertexProgramStrategyTest {
final ComputerResultStep computerResultStep = new ComputerResultStep(EmptyTraversal.instance());
return Arrays.asList(new Traversal[][]{
+ { EmptyGraph.instance().traversal().read("blah.json"), EmptyGraph.instance().traversal().read("blah.json")},
+ { EmptyGraph.instance().traversal().write("blah.json"), EmptyGraph.instance().traversal().write("blah.json")},
{__.V().out().count(), start().addStep(traversal(__.V().out().count())).addStep(computerResultStep)},
{__.V().pageRank().out().count(), start().pageRank().asAdmin().addStep(traversal(__.V().out().count())).addStep(computerResultStep)},
{__.V().out().pageRank(), start().addStep(traversal(__.V().out())).pageRank().asAdmin().addStep(traversal(__.identity())).addStep(computerResultStep)},
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java
index f7c19ac..43da8b7 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java
@@ -63,6 +63,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProjectTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest;
@@ -153,6 +154,7 @@ public class ProcessStandardSuite extends AbstractGremlinSuite {
ProfileTest.Traversals.class,
ProjectTest.Traversals.class,
PropertiesTest.Traversals.class,
+ ReadTest.Traversals.class,
SelectTest.Traversals.class,
VertexTest.Traversals.class,
UnfoldTest.Traversals.class,
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
new file mode 100644
index 0000000..2cfc3f7
--- /dev/null
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.traversal.step.map;
+
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
+import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
+import org.apache.tinkerpop.gremlin.process.IgnoreEngine;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.IoTest;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+@RunWith(GremlinProcessRunner.class)
+public abstract class ReadTest extends AbstractGremlinProcessTest {
+
+ public abstract Traversal<Map<String,Object>, Map<String,Object>> get_g_read() throws IOException;
+
+ @Test
+ public void g_read() throws IOException {
+ final Traversal<Map<String,Object>, Map<String,Object>> traversal = get_g_read();
+ printTraversalForm(traversal);
+ assertTrue(traversal.hasNext());
+
+ IoTest.assertModernGraph(graph, false, true);
+ }
+
+ public static class Traversals extends ReadTest {
+ @Override
+ public Traversal<Map<String,Object>, Map<String,Object>> get_g_read() throws IOException {
+ final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GryoResourceAccess.class, "tinkerpop-modern-v3d0.kryo", "").getAbsolutePath().replace('\\', '/');
+ return g.read(fileToRead);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopReadStep.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopReadStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopReadStep.java
new file mode 100644
index 0000000..c896d76
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopReadStep.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Reading;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public class HadoopReadStep extends VertexProgramStep implements Reading {
+
+ private Parameters parameters = new Parameters();
+
+ public HadoopReadStep(final Traversal.Admin traversal, final String localFile) {
+ super(traversal);
+
+ final Graph graph = (Graph) traversal.getGraph().get();
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, "org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat");
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, localFile);
+ }
+
+ @Override
+ public String getLocalFile() {
+ return (String) ((Graph) traversal.getGraph().get()).configuration().getProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ }
+
+ @Override
+ public void configure(final Object... keyValues) {
+ // TODO: probably should write to the Configuration selectively - no need for actual Parameters?????????
+ this.parameters.set(null, keyValues);
+ }
+
+ @Override
+ public Parameters getParameters() {
+ return parameters;
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.stepString(this, new GraphFilter(this.computer));
+ }
+
+ @Override
+ public CloneVertexProgram generateProgram(final Graph graph, final Memory memory) {
+ return CloneVertexProgram.build().create(graph);
+ }
+
+ @Override
+ public HadoopReadStep clone() {
+ return (HadoopReadStep) super.clone();
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopWriteStep.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopWriteStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopWriteStep.java
new file mode 100644
index 0000000..0711a2e
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopWriteStep.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Reading;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public class HadoopWriteStep extends VertexProgramStep implements Reading {
+
+ private Parameters parameters = new Parameters();
+
+ public HadoopWriteStep(final Traversal.Admin traversal, final String localFile) {
+ super(traversal);
+
+ final Graph graph = (Graph) traversal.getGraph().get();
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, "org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat");
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, localFile);
+ }
+
+ @Override
+ public String getLocalFile() {
+ return (String) ((Graph) traversal.getGraph().get()).configuration().getProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
+ }
+
+ @Override
+ public void configure(final Object... keyValues) {
+ // TODO: probably should write to the Configuration selectively - no need for actual Parameters?????????
+ this.parameters.set(null, keyValues);
+ }
+
+ @Override
+ public Parameters getParameters() {
+ return parameters;
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.stepString(this, new GraphFilter(this.computer));
+ }
+
+ @Override
+ public CloneVertexProgram generateProgram(final Graph graph, final Memory memory) {
+ return CloneVertexProgram.build().create(graph);
+ }
+
+ @Override
+ public HadoopWriteStep clone() {
+ return (HadoopWriteStep) super.clone();
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
new file mode 100644
index 0000000..47986f9
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.strategy;
+
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map.HadoopReadStep;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map.HadoopWriteStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Reading;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Writing;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public final class HadoopIoStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy>
+ implements TraversalStrategy.ProviderOptimizationStrategy {
+
+ private static final HadoopIoStrategy INSTANCE = new HadoopIoStrategy();
+
+ private HadoopIoStrategy() {
+ }
+
+ @Override
+ public void apply(final Traversal.Admin<?, ?> traversal) {
+ // replace Reading and Writing steps with hadoop specific ones
+ if (traversal.getStartStep() instanceof Reading) {
+ final Reading reading = (Reading) traversal.getStartStep();
+ final HadoopReadStep hadoopReadStep = new HadoopReadStep(traversal, reading.getLocalFile());
+ reading.getParameters().getRaw().entrySet().forEach(kv ->
+ hadoopReadStep.configure(null, kv.getKey(), kv.getValue())
+ );
+
+ TraversalHelper.replaceStep((Step) reading, hadoopReadStep, traversal);
+ } else if (traversal.getStartStep() instanceof Writing) {
+ final Writing writing = (Writing) traversal.getStartStep();
+ final HadoopWriteStep hadoopWriteStep = new HadoopWriteStep(traversal, writing.getLocalFile());
+ writing.getParameters().getRaw().entrySet().forEach(kv ->
+ hadoopWriteStep.configure(null, kv.getKey(), kv.getValue())
+ );
+
+ TraversalHelper.replaceStep((Step) writing, hadoopWriteStep, traversal);
+ }
+ }
+
+ public static HadoopIoStrategy instance() {
+ return INSTANCE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index ea8334b..14c5360 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -24,9 +24,11 @@ import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.strategy.HadoopIoStrategy;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopEdgeIterator;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopVertexIterator;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Transaction;
@@ -148,6 +150,12 @@ public final class HadoopGraph implements Graph {
this.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
}};
+ static {
+ TraversalStrategies.GlobalCache.registerStrategies(HadoopGraph.class,
+ TraversalStrategies.GlobalCache.getStrategies(Graph.class).clone().addStrategies(
+ HadoopIoStrategy.instance()));
+ }
+
protected final HadoopConfiguration configuration;
private HadoopGraph(final Configuration configuration) {