You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2016/05/31 12:49:53 UTC
[46/49] incubator-tinkerpop git commit: added BulkExportVertexProgram
added BulkExportVertexProgram
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/df6cfd60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/df6cfd60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/df6cfd60
Branch: refs/heads/TINKERPOP-1298
Commit: df6cfd603468e05a80070b2ba547059c5f4fec12
Parents: 670bc69
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Sun May 22 09:37:41 2016 +0200
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Tue May 31 14:47:25 2016 +0200
----------------------------------------------------------------------
.../bulkdumping/BulkExportVertexProgram.java | 163 +++++++++++++++++++
.../gremlin/structure/io/gryo/GryoMapper.java | 4 +-
2 files changed, 166 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/df6cfd60/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkExportVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkExportVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkExportVertexProgram.java
new file mode 100644
index 0000000..cdd6b28
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkExportVertexProgram.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.computer.bulkdumping;
+
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationUtils;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
+import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.computer.VertexComputeKey;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.AbstractVertexProgramBuilder;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+import org.javatuples.Tuple;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * @author Daniel Kuppitz (http://gremlin.guru)
+ */
+public class BulkExportVertexProgram implements VertexProgram<Tuple> {
+
+ public static final String BULK_EXPORT_VERTEX_PROGRAM_CFG_PREFIX = "gremlin.bulkExportVertexProgram";
+ public static final String BULK_EXPORT_PROPERTIES = String.join(".", BULK_EXPORT_VERTEX_PROGRAM_CFG_PREFIX, "properties");
+
+ private Configuration configuration;
+ private String properties;
+ private Set<VertexComputeKey> vertexComputeKeys;
+
+ private BulkExportVertexProgram() {
+ }
+
+ @Override
+ public void loadState(final Graph graph, final Configuration config) {
+ configuration = new BaseConfiguration();
+ if (config != null) {
+ ConfigurationUtils.copy(config, configuration);
+ }
+ properties = configuration.getString(BULK_EXPORT_PROPERTIES);
+ vertexComputeKeys = Collections.singleton(VertexComputeKey.of(BULK_EXPORT_PROPERTIES, false));
+ }
+
+ @Override
+ public void storeState(final Configuration config) {
+ VertexProgram.super.storeState(config);
+ if (configuration != null) {
+ ConfigurationUtils.copy(configuration, config);
+ }
+ }
+
+ @Override
+ public void setup(final Memory memory) {
+ }
+
+ @Override
+ public void execute(final Vertex sourceVertex, final Messenger<Tuple> messenger, final Memory memory) {
+ final VertexProperty<TraverserSet> haltedTraversers = sourceVertex.property(TraversalVertexProgram.HALTED_TRAVERSERS);
+ if (haltedTraversers.isPresent()) {
+ sourceVertex.property(BULK_EXPORT_PROPERTIES, properties);
+ }
+ }
+
+ @Override
+ public boolean terminate(final Memory memory) {
+ return properties == null || properties.isEmpty() || !memory.isInitialIteration();
+ }
+
+ @Override
+ public Set<MessageScope> getMessageScopes(final Memory memory) {
+ return Collections.emptySet();
+ }
+
+ @SuppressWarnings({"CloneDoesntDeclareCloneNotSupportedException", "CloneDoesntCallSuperClone"})
+ @Override
+ public VertexProgram<Tuple> clone() {
+ return this;
+ }
+
+ @Override
+ public GraphComputer.ResultGraph getPreferredResultGraph() {
+ return GraphComputer.ResultGraph.NEW;
+ }
+
+ @Override
+ public GraphComputer.Persist getPreferredPersist() {
+ return GraphComputer.Persist.EDGES;
+ }
+
+ @Override
+ public Set<VertexComputeKey> getVertexComputeKeys() {
+ return this.vertexComputeKeys;
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.vertexProgramString(this);
+ }
+
+ public static Builder build() {
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractVertexProgramBuilder<Builder> {
+
+ private List<String> properties;
+
+ private Builder() {
+ super(BulkExportVertexProgram.class);
+ properties = new ArrayList<>();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public BulkExportVertexProgram create(final Graph graph) {
+ configuration.setProperty(BULK_EXPORT_PROPERTIES, String.join("\1", properties));
+ graph.configuration().setProperty("gremlin.hadoop.graphWriter", "org.apache.tinkerpop.gremlin.hadoop.structure.io.script.ScriptOutputFormat");
+ return (BulkExportVertexProgram) VertexProgram.createVertexProgram(graph, configuration);
+ }
+
+ public Builder key(final String key, final String format) {
+ properties.add(key + "\2" + format);
+ return this;
+ }
+
+ public Builder key(final String key) {
+ this.key(key, "");
+ return this;
+ }
+
+ public Builder keys(final String... keys) {
+ for (final String key : keys) {
+ this.key(key, "");
+ }
+ return this;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/df6cfd60/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index 9cae845..705e136 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -95,6 +95,7 @@ import java.time.Year;
import java.time.YearMonth;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
+import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
@@ -295,6 +296,7 @@ public final class GryoMapper implements Mapper<Kryo> {
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(UUID.class, kryo -> new UUIDSerializer(), 17));
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(URI.class, kryo -> new URISerializer(), 72));
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(VertexTerminator.class, null, 13));
+ add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(AbstractMap.SimpleEntry.class, kryo -> new EntrySerializer(), 120)); // ***LAST ID***
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ReferenceEdge.class, null, 81));
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ReferenceVertexProperty.class, null, 82));
@@ -356,7 +358,7 @@ public final class GryoMapper implements Mapper<Kryo> {
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(GroupStepV3d0.GroupBiOperatorV3d0.class, null, 113));
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(RangeGlobalStep.RangeBiOperator.class, null, 114));
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(OrderGlobalStep.OrderBiOperator.class, kryo -> new JavaSerializer(), 118)); // because they contain traversals
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ProfileStep.ProfileBiOperator.class, null, 119)); // ***LAST ID***
+ add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ProfileStep.ProfileBiOperator.class, null, 119));
}};
private final List<IoRegistry> registries = new ArrayList<>();