You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/04 15:42:11 UTC
[16/20] incubator-tinkerpop git commit: OMG -- Apache Configuration
is the wooooorst --- the auto , deliminator ... total hole for the last hour.
OMG -- Apache Configuration is the wooooorst --- the auto , deliminator ... total hole for the last hour.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/a9d0cf15
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/a9d0cf15
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/a9d0cf15
Branch: refs/heads/master
Commit: a9d0cf15602eaecf89b556e9c2783fb60ca0c847
Parents: b045e61
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 3 16:26:00 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 3 16:26:00 2015 -0700
----------------------------------------------------------------------
.../computer/util/VertexProgramHelper.java | 18 +++--
.../computer/giraph/GiraphGraphComputer.java | 1 -
.../spark/SerializableConfiguration.java | 69 -------------------
.../computer/spark/SparkGraphComputer.java | 16 ++---
.../spark/util/SApacheConfiguration.java | 70 ++++++++++++++++++++
5 files changed, 89 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a9d0cf15/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
index c7c37b8..88c22b4 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
@@ -18,13 +18,14 @@
*/
package org.apache.tinkerpop.gremlin.process.computer.util;
+import org.apache.commons.configuration.AbstractConfiguration;
+import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.process.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
import org.apache.tinkerpop.gremlin.util.Serializer;
-import org.apache.commons.configuration.Configuration;
import java.io.IOException;
-import java.util.List;
+import java.util.Arrays;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -35,8 +36,11 @@ public final class VertexProgramHelper {
}
public static void serialize(final Object object, final Configuration configuration, final String key) {
+ if (configuration instanceof AbstractConfiguration)
+ ((AbstractConfiguration) configuration).setDelimiterParsingDisabled(true);
try {
- configuration.setProperty(key, Serializer.serializeObject(object));
+ final String byteString = Arrays.toString(Serializer.serializeObject(object));
+ configuration.setProperty(key, byteString.substring(1, byteString.length() - 1));
} catch (final IOException e) {
throw new IllegalArgumentException(e.getMessage(), e);
}
@@ -44,10 +48,10 @@ public final class VertexProgramHelper {
public static <T> T deserialize(final Configuration configuration, final String key) {
try {
- final List byteList = configuration.getList(key);
- byte[] bytes = new byte[byteList.size()];
- for (int i = 0; i < byteList.size(); i++) {
- bytes[i] = Byte.valueOf(byteList.get(i).toString().replace("[", "").replace("]", ""));
+ final String[] stringBytes = configuration.getString(key).split(",");
+ byte[] bytes = new byte[stringBytes.length];
+ for (int i = 0; i < stringBytes.length; i++) {
+ bytes[i] = Byte.valueOf(stringBytes[i].trim());
}
return (T) Serializer.deserializeObject(bytes);
} catch (final IOException | ClassNotFoundException e) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a9d0cf15/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
index 56d029c..52d606c 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
@@ -140,7 +140,6 @@ public class GiraphGraphComputer extends Configured implements GraphComputer, To
this.loadJars(fs);
fs.delete(new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION)), true);
ToolRunner.run(this, new String[]{});
- // memory.keys().forEach(k -> LOGGER.error(k + "---" + memory.get(k)));
} catch (Exception e) {
//e.printStackTrace();
throw new IllegalStateException(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a9d0cf15/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SerializableConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SerializableConfiguration.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SerializableConfiguration.java
deleted file mode 100644
index 73e3d08..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SerializableConfiguration.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.commons.configuration.AbstractConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.ConfigurationUtils;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SerializableConfiguration extends AbstractConfiguration implements Serializable {
-
- private final Map<String, Object> configurations = new HashMap<>();
-
- public SerializableConfiguration() {
-
- }
-
- public SerializableConfiguration(final Configuration configuration) {
- ConfigurationUtils.copy(configuration, this);
- }
-
- @Override
- protected void addPropertyDirect(final String key, final Object value) {
- this.configurations.put(key, value);
- }
-
- @Override
- public boolean isEmpty() {
- return this.configurations.isEmpty();
- }
-
- @Override
- public boolean containsKey(final String key) {
- return this.configurations.containsKey(key);
- }
-
- @Override
- public Object getProperty(final String key) {
- return this.configurations.get(key);
- }
-
- @Override
- public Iterator<String> getKeys() {
- return this.configurations.keySet().iterator();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a9d0cf15/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
index dd004bc..c089c57 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
@@ -29,6 +29,7 @@ import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.util.SApacheConfiguration;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.util.SparkHelper;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
@@ -112,8 +113,8 @@ public final class SparkGraphComputer implements GraphComputer {
this.mapReducers.addAll(this.vertexProgram.getMapReducers());
}
// apache and hadoop configurations that are used throughout
- final org.apache.commons.configuration.Configuration apacheConfiguration = this.hadoopGraph.configuration();
- final Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(this.hadoopGraph.configuration());
+ final org.apache.commons.configuration.Configuration apacheConfiguration = new SApacheConfiguration(this.hadoopGraph.configuration());
+ final Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(apacheConfiguration);
return CompletableFuture.<ComputerResult>supplyAsync(() -> {
final long startTime = System.currentTimeMillis();
@@ -143,15 +144,14 @@ public final class SparkGraphComputer implements GraphComputer {
// set up the vertex program and wire up configurations
memory = new SparkMemory(this.vertexProgram, this.mapReducers, sparkContext);
this.vertexProgram.setup(memory);
- final SerializableConfiguration vertexProgramConfiguration = new SerializableConfiguration();
+ final SApacheConfiguration vertexProgramConfiguration = new SApacheConfiguration();
this.vertexProgram.storeState(vertexProgramConfiguration);
- ConfUtil.mergeApacheIntoHadoopConfiguration(vertexProgramConfiguration, hadoopConfiguration);
ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration);
-
+ ConfUtil.mergeApacheIntoHadoopConfiguration(vertexProgramConfiguration, hadoopConfiguration);
// execute the vertex program
do {
graphRDD = SparkHelper.executeStep(graphRDD, this.vertexProgram, memory, vertexProgramConfiguration);
- graphRDD.foreachPartition(iterator -> doNothing()); // i think this is a fast way to execute the rdd
+ graphRDD.foreachPartition(iterator -> doNothing()); // TODO: i think this is a fast way to execute the rdd
graphRDD.cache(); // TODO: learn about persistence and caching
memory.incrIteration();
} while (!this.vertexProgram.terminate(memory));
@@ -185,7 +185,7 @@ public final class SparkGraphComputer implements GraphComputer {
NullWritable.class,
VertexWritable.class);
- final SerializableConfiguration newApacheConfiguration = new SerializableConfiguration(apacheConfiguration);
+ final SApacheConfiguration newApacheConfiguration = new SApacheConfiguration(apacheConfiguration);
mapReduce.storeState(newApacheConfiguration);
// map
final JavaPairRDD mapRDD = SparkHelper.executeMap(hadoopGraphRDD, mapReduce, newApacheConfiguration);
@@ -243,7 +243,7 @@ public final class SparkGraphComputer implements GraphComputer {
return new Features() {
@Override
public boolean supportsNonSerializableObjects() {
- return true; // TODO
+ return false;
}
};
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a9d0cf15/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SApacheConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SApacheConfiguration.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SApacheConfiguration.java
new file mode 100644
index 0000000..dbd468a
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SApacheConfiguration.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.util;
+
+import org.apache.commons.configuration.AbstractConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationUtils;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SApacheConfiguration extends AbstractConfiguration implements Serializable {
+
+ private final Map<String, Object> configurations = new HashMap<>();
+
+ public SApacheConfiguration() {
+ this.setDelimiterParsingDisabled(true); // gets me everytime (what a stupid default behavior)
+ }
+
+ public SApacheConfiguration(final Configuration configuration) {
+ this();
+ ConfigurationUtils.copy(configuration, this);
+ }
+
+ @Override
+ protected void addPropertyDirect(final String key, final Object value) {
+ this.configurations.put(key, value);
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return this.configurations.isEmpty();
+ }
+
+ @Override
+ public boolean containsKey(final String key) {
+ return this.configurations.containsKey(key);
+ }
+
+ @Override
+ public Object getProperty(final String key) {
+ return this.configurations.get(key);
+ }
+
+ @Override
+ public Iterator<String> getKeys() {
+ return this.configurations.keySet().iterator();
+ }
+}