You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2018/08/01 19:29:40 UTC
[16/50] [abbrv] tinkerpop git commit: TINKERPOP-1996 Pass
configurations from with() through to Hadoop
TINKERPOP-1996 Pass configurations from with() through to Hadoop
This will allow users to override or add to the Hadoop/Spark/OLAP configuration as needed
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/4d979cf8
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/4d979cf8
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/4d979cf8
Branch: refs/heads/TINKERPOP-1990
Commit: 4d979cf8dc4482d574191a76a0c60e281f887563
Parents: 51dc821
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 07:35:48 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 07:35:48 2018 -0400
----------------------------------------------------------------------
.../traversal/step/sideEffect/HadoopIoStep.java | 29 ++++++++++++++++++--
1 file changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/4d979cf8/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
index ca369b6..2e96276 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
@@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
@@ -37,7 +38,9 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
/**
* An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)} which uses the
- * {@link CloneVertexProgram} for its implementation.
+ * {@link CloneVertexProgram} for its implementation. Standard Hadoop OLAP configurations can be passed using the
+ * {@link GraphTraversal#with(String, Object)} step modulator as all options aside from those in {@link IO} will be
+ * transferred.
*
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
@@ -69,7 +72,6 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting {
@Override
public void configure(final Object... keyValues) {
- // TODO: probably should write to the Configuration selectively - no need for actual Parameters?????????
this.parameters.set(null, keyValues);
}
@@ -121,6 +123,8 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting {
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName);
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file);
+
+ addParametersToConfiguration(graph);
}
private void configureForWrite(final Graph graph) {
@@ -137,6 +141,27 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting {
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName);
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file);
+
+ addParametersToConfiguration(graph);
+ }
+
+ /**
+ * Overwrites all configurations from values passed using {@link GraphTraversal#with(String, Object)}.
+ */
+ private void addParametersToConfiguration(final Graph graph) {
+ parameters.getRaw(IO.writer, IO.writer, IO.registry).entrySet().forEach(kv -> {
+ if (kv.getValue().size() == 1)
+ graph.configuration().setProperty(kv.getKey().toString(), kv.getValue().get(0));
+ else {
+ // reset the default configuration with the first option then add to that for List options
+ for (int ix = 0; ix < kv.getValue().size(); ix++) {
+ if (ix == 0)
+ graph.configuration().setProperty(kv.getKey().toString(), kv.getValue().get(ix));
+ else
+ graph.configuration().addProperty(kv.getKey().toString(), kv.getValue().get(ix));
+ }
+ }
+ });
}
private String detectReader() {