You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/03/29 21:01:44 UTC
[1/8] incubator-tinkerpop git commit: If the HADOOP_GREMLIN_LIBS path
does not exist locally, then search for it in HDFS. If it exists there,
copy it to a java.io.tmp directory and then pull out the jars from there to
upload to the DistributdCache (Girap
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 8a1d8a1e0 -> b78ffcf3c
If the HADOOP_GREMLIN_LIBS path does not exist locally, then search for it in HDFS. If it exists there, copy it to a java.io.tmp directory and then pull out the jars from there to upload to the DistributdCache (Giraph) or SparkServer (Spark).
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/c6b9bc71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/c6b9bc71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/c6b9bc71
Branch: refs/heads/master
Commit: c6b9bc7114aa77b69edf942486517c04794c7d45
Parents: da8c5fb
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Mar 28 14:18:01 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Mar 28 14:18:01 2016 -0600
----------------------------------------------------------------------
.../process/computer/GiraphGraphComputer.java | 2 +-
.../computer/AbstractHadoopGraphComputer.java | 30 +++++++
.../AbstractHadoopGraphComputerTest.java | 82 ++++++++++++++++++++
.../process/computer/SparkGraphComputer.java | 19 +++--
4 files changed, 125 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c6b9bc71/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
index 76eda1d..3ed8618 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
@@ -247,7 +247,7 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
else {
final String[] paths = hadoopGremlinLibsLocal.split(":");
for (final String path : paths) {
- final File file = new File(path);
+ final File file = AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, path);
if (file.exists()) {
Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> {
try {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c6b9bc71/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
index b766089..7659f0d 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
@@ -18,6 +18,10 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.process.computer;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
@@ -31,9 +35,12 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+import org.apache.tinkerpop.gremlin.util.Gremlin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
@@ -195,4 +202,27 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
return false;
}
}
+
+ //////////
+
+ public static File copyDirectoryIfNonExistent(final FileSystem fileSystem, final String localDirectory) {
+ try {
+ final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
+ File file = new File(localDirectory);
+ if ((Boolean.valueOf(System.getProperty("is.testing", "false")) || !file.exists()) && fileSystem.exists(new Path(localDirectory)) && fileSystem.isDirectory(new Path(localDirectory))) {
+ final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
+ if (!tempDirectory.exists()) assert tempDirectory.mkdir();
+ final String tempPath = tempDirectory.getAbsolutePath() + "/" + new File(localDirectory).getName();
+ final RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path(localDirectory), false);
+ while (files.hasNext()) {
+ final LocatedFileStatus f = files.next();
+ fileSystem.copyToLocalFile(f.getPath(), new Path(tempPath + "/" + f.getPath().getName()));
+ }
+ return new File(tempPath);
+ } else
+ return file;
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c6b9bc71/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
new file mode 100644
index 0000000..99916db
--- /dev/null
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.hadoop.process.computer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tinkerpop.gremlin.util.Gremlin;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class AbstractHadoopGraphComputerTest {
+
+ @Test
+ public void shouldCopyDirectoriesCorrectly() throws Exception {
+ final FileSystem fs = FileSystem.get(new Configuration());
+ if (!new File(System.getProperty("java.io.tmpdir") + "/tmp").exists())
+ assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").mkdir());
+ File tempFile1 = new File(System.getProperty("java.io.tmpdir") + "/tmp/test1.txt");
+ File tempFile2 = new File(System.getProperty("java.io.tmpdir") + "/tmp/test2.txt");
+ assertTrue(tempFile1.createNewFile());
+ assertTrue(tempFile2.createNewFile());
+ assertTrue(tempFile1.exists());
+ assertTrue(tempFile2.exists());
+ if (fs.exists(new Path("target/testing")))
+ assertTrue(fs.delete(new Path("target/testing"), true));
+ fs.copyFromLocalFile(true, new Path(tempFile1.getAbsolutePath()), new Path("target/testing/test1.dat"));
+ fs.copyFromLocalFile(true, new Path(tempFile2.getAbsolutePath()), new Path("target/testing/test2.dat"));
+ assertTrue(fs.exists(new Path("target/testing/test1.dat")));
+ assertTrue(fs.exists(new Path("target/testing/test2.dat")));
+ assertTrue(fs.exists(new Path("target/testing")));
+ assertTrue(fs.isDirectory(new Path("target/testing")));
+ assertFalse(tempFile1.exists());
+ assertFalse(tempFile2.exists());
+ assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").exists());
+ assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").delete());
+ assertTrue(fs.exists(new Path("target/testing/test1.dat")));
+ assertTrue(fs.exists(new Path("target/testing/test2.dat")));
+ assertTrue(fs.exists(new Path("target/testing")));
+ assertTrue(fs.isDirectory(new Path("target/testing")));
+ /////
+ final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
+ final File localDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
+ final File localLibDirectory = new File(localDirectory.getAbsolutePath() + "/testing");
+ assertTrue(localDirectory.exists());
+ if (localLibDirectory.exists()) {
+ Stream.of(localLibDirectory.listFiles()).forEach(File::delete);
+ assertTrue(localLibDirectory.delete());
+ }
+ assertFalse(localLibDirectory.exists());
+ assertEquals(localLibDirectory, AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, "target/testing"));
+ assertTrue(localLibDirectory.exists());
+ assertTrue(localLibDirectory.isDirectory());
+ assertEquals(2, Stream.of(localLibDirectory.listFiles()).filter(file -> file.getName().endsWith(".dat")).count());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c6b9bc71/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index f3b2eb9..928a880 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -323,13 +323,18 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
if (null == hadoopGremlinLocalLibs)
this.logger.warn(Constants.HADOOP_GREMLIN_LIBS + " is not set -- proceeding regardless");
else {
- final String[] paths = hadoopGremlinLocalLibs.split(":");
- for (final String path : paths) {
- final File file = new File(path);
- if (file.exists())
- Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> sparkContext.addJar(f.getAbsolutePath()));
- else
- this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
+ try {
+ final String[] paths = hadoopGremlinLocalLibs.split(":");
+ final FileSystem fs = FileSystem.get(hadoopConfiguration);
+ for (final String path : paths) {
+ final File file = AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, path);
+ if (file.exists())
+ Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> sparkContext.addJar(f.getAbsolutePath()));
+ else
+ this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
+ }
+ } catch (IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
}
}
}
[7/8] incubator-tinkerpop git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-tinkerpop
Posted by ok...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-tinkerpop
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/70b7fc35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/70b7fc35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/70b7fc35
Branch: refs/heads/master
Commit: 70b7fc35b1282fa262559e6c1c0ddb595d1d8bdd
Parents: ea30b1c 8a1d8a1
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 29 13:00:53 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 29 13:00:53 2016 -0600
----------------------------------------------------------------------
.../src/reference/gremlin-applications.asciidoc | 3 ++
.../gremlin/groovy/engine/GremlinExecutor.java | 27 +++++++++++++++---
.../groovy/engine/GremlinExecutorTest.java | 29 ++++++++++++++++++++
.../op/traversal/TraversalOpProcessor.java | 12 +++++++-
4 files changed, 66 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[3/8] incubator-tinkerpop git commit: made mkdir() to mkdirs() per
@dkuppitz's request.
Posted by ok...@apache.org.
made mkdir() to mkdirs() per @dkuppitz's request.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/da791aa9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/da791aa9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/da791aa9
Branch: refs/heads/master
Commit: da791aa9b9c772050cf6c7797f28622b7663f1dc
Parents: cbd8200
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Mar 28 18:17:46 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Mar 28 18:17:46 2016 -0600
----------------------------------------------------------------------
.../hadoop/process/computer/AbstractHadoopGraphComputer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/da791aa9/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
index 7659f0d..28dd7e5 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
@@ -211,7 +211,7 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
File file = new File(localDirectory);
if ((Boolean.valueOf(System.getProperty("is.testing", "false")) || !file.exists()) && fileSystem.exists(new Path(localDirectory)) && fileSystem.isDirectory(new Path(localDirectory))) {
final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
- if (!tempDirectory.exists()) assert tempDirectory.mkdir();
+ if (!tempDirectory.exists()) assert tempDirectory.mkdirs();
final String tempPath = tempDirectory.getAbsolutePath() + "/" + new File(localDirectory).getName();
final RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path(localDirectory), false);
while (files.hasNext()) {
[4/8] incubator-tinkerpop git commit: added
System.getProperty(file.separator)
Posted by ok...@apache.org.
added System.getProperty(file.separator)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/79009e60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/79009e60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/79009e60
Branch: refs/heads/master
Commit: 79009e60d5f35cb4fc92801b23ccc1edec8662cf
Parents: da791aa
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 29 08:46:42 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 29 08:46:42 2016 -0600
----------------------------------------------------------------------
.../hadoop/process/computer/AbstractHadoopGraphComputer.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/79009e60/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
index 28dd7e5..a05a1be 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
@@ -210,13 +210,13 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
File file = new File(localDirectory);
if ((Boolean.valueOf(System.getProperty("is.testing", "false")) || !file.exists()) && fileSystem.exists(new Path(localDirectory)) && fileSystem.isDirectory(new Path(localDirectory))) {
- final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
+ final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + hadoopGremlinLibsRemote);
if (!tempDirectory.exists()) assert tempDirectory.mkdirs();
- final String tempPath = tempDirectory.getAbsolutePath() + "/" + new File(localDirectory).getName();
+ final String tempPath = tempDirectory.getAbsolutePath() + System.getProperty("file.separator") + new File(localDirectory).getName();
final RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path(localDirectory), false);
while (files.hasNext()) {
final LocatedFileStatus f = files.next();
- fileSystem.copyToLocalFile(f.getPath(), new Path(tempPath + "/" + f.getPath().getName()));
+ fileSystem.copyToLocalFile(f.getPath(), new Path(tempPath + System.getProperty("file.separator") + f.getPath().getName()));
}
return new File(tempPath);
} else
[2/8] incubator-tinkerpop git commit: dar,
test was bad. had residual directories that were not cleaned up. Thus,
on a fresh system, it failed.
Posted by ok...@apache.org.
dar, test was bad. had residual directories that were not cleaned up. Thus, on a fresh system, it failed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/cbd8200a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/cbd8200a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/cbd8200a
Branch: refs/heads/master
Commit: cbd8200aa26ced615ecd6c1d8eb007ea3417b634
Parents: c6b9bc7
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Mar 28 15:00:24 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Mar 28 15:00:24 2016 -0600
----------------------------------------------------------------------
.../hadoop/process/computer/AbstractHadoopGraphComputerTest.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/cbd8200a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
index 99916db..4a2c8a9 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
@@ -68,7 +68,6 @@ public class AbstractHadoopGraphComputerTest {
final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
final File localDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
final File localLibDirectory = new File(localDirectory.getAbsolutePath() + "/testing");
- assertTrue(localDirectory.exists());
if (localLibDirectory.exists()) {
Stream.of(localLibDirectory.listFiles()).forEach(File::delete);
assertTrue(localLibDirectory.delete());
[8/8] incubator-tinkerpop git commit: updated CHANGELOG.
Posted by ok...@apache.org.
updated CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b78ffcf3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b78ffcf3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b78ffcf3
Branch: refs/heads/master
Commit: b78ffcf3c47ae135abb23258e6bc465393dd06ab
Parents: 70b7fc3
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 29 13:01:35 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 29 13:01:35 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b78ffcf3/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 87a7ba6..597e27d 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.2.0 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* `HADOOP_GREMLIN_LIBS` can now reference a directory in HDFS and will be used if the directory does not exist locally.
* Added `gremlin-benchmark` module with JMH benchmarking base classes that can be used for further benchmark development.
* `TraversalStrategies.GlobalCache` supports both `Graph` and `GraphComputer` strategy registrations.
* `select("a","b").by("name").by("age")`-style traversals now work in OLAP with new `PathProcessorStrategy`.
[5/8] incubator-tinkerpop git commit: used testclassname-based
directories to reduce the probability of target/ directory conflicts.
Posted by ok...@apache.org.
used testclassname-based directories to reduce the probability of target/ directory conflicts.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/bef52c61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/bef52c61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/bef52c61
Branch: refs/heads/master
Commit: bef52c611fa1484774df02d049d03bb06f7d07f4
Parents: 79009e6
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 29 08:51:42 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 29 08:51:42 2016 -0600
----------------------------------------------------------------------
.../AbstractHadoopGraphComputerTest.java | 42 ++++++++++----------
1 file changed, 22 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bef52c61/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
index 4a2c8a9..56f0417 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
@@ -39,41 +39,43 @@ public class AbstractHadoopGraphComputerTest {
@Test
public void shouldCopyDirectoriesCorrectly() throws Exception {
+ final String hdfsName = this.getClass().getSimpleName() + "-hdfs";
+ final String localName = this.getClass().getSimpleName() + "-local";
final FileSystem fs = FileSystem.get(new Configuration());
- if (!new File(System.getProperty("java.io.tmpdir") + "/tmp").exists())
- assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").mkdir());
- File tempFile1 = new File(System.getProperty("java.io.tmpdir") + "/tmp/test1.txt");
- File tempFile2 = new File(System.getProperty("java.io.tmpdir") + "/tmp/test2.txt");
+ if (!new File(System.getProperty("java.io.tmpdir") + "/" + localName).exists())
+ assertTrue(new File(System.getProperty("java.io.tmpdir") + "/" + localName).mkdir());
+ File tempFile1 = new File(System.getProperty("java.io.tmpdir") + "/" + localName + "/test1.txt");
+ File tempFile2 = new File(System.getProperty("java.io.tmpdir") + "/" + localName + "/test2.txt");
assertTrue(tempFile1.createNewFile());
assertTrue(tempFile2.createNewFile());
assertTrue(tempFile1.exists());
assertTrue(tempFile2.exists());
- if (fs.exists(new Path("target/testing")))
- assertTrue(fs.delete(new Path("target/testing"), true));
- fs.copyFromLocalFile(true, new Path(tempFile1.getAbsolutePath()), new Path("target/testing/test1.dat"));
- fs.copyFromLocalFile(true, new Path(tempFile2.getAbsolutePath()), new Path("target/testing/test2.dat"));
- assertTrue(fs.exists(new Path("target/testing/test1.dat")));
- assertTrue(fs.exists(new Path("target/testing/test2.dat")));
- assertTrue(fs.exists(new Path("target/testing")));
- assertTrue(fs.isDirectory(new Path("target/testing")));
+ if (fs.exists(new Path("target/" + hdfsName)))
+ assertTrue(fs.delete(new Path("target/" + hdfsName), true));
+ fs.copyFromLocalFile(true, new Path(tempFile1.getAbsolutePath()), new Path("target/" + hdfsName + "/test1.dat"));
+ fs.copyFromLocalFile(true, new Path(tempFile2.getAbsolutePath()), new Path("target/" + hdfsName + "/test2.dat"));
+ assertTrue(fs.exists(new Path("target/" + hdfsName + "/test1.dat")));
+ assertTrue(fs.exists(new Path("target/" + hdfsName + "/test2.dat")));
+ assertTrue(fs.exists(new Path("target/" + hdfsName)));
+ assertTrue(fs.isDirectory(new Path("target/" + hdfsName)));
assertFalse(tempFile1.exists());
assertFalse(tempFile2.exists());
- assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").exists());
- assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").delete());
- assertTrue(fs.exists(new Path("target/testing/test1.dat")));
- assertTrue(fs.exists(new Path("target/testing/test2.dat")));
- assertTrue(fs.exists(new Path("target/testing")));
- assertTrue(fs.isDirectory(new Path("target/testing")));
+ assertTrue(new File(System.getProperty("java.io.tmpdir") + "/" + localName).exists());
+ assertTrue(new File(System.getProperty("java.io.tmpdir") + "/" + localName).delete());
+ assertTrue(fs.exists(new Path("target/" + hdfsName + "/test1.dat")));
+ assertTrue(fs.exists(new Path("target/" + hdfsName + "/test2.dat")));
+ assertTrue(fs.exists(new Path("target/" + hdfsName)));
+ assertTrue(fs.isDirectory(new Path("target/" + hdfsName)));
/////
final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
final File localDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
- final File localLibDirectory = new File(localDirectory.getAbsolutePath() + "/testing");
+ final File localLibDirectory = new File(localDirectory.getAbsolutePath() + "/" + hdfsName);
if (localLibDirectory.exists()) {
Stream.of(localLibDirectory.listFiles()).forEach(File::delete);
assertTrue(localLibDirectory.delete());
}
assertFalse(localLibDirectory.exists());
- assertEquals(localLibDirectory, AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, "target/testing"));
+ assertEquals(localLibDirectory, AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, "target/" + hdfsName));
assertTrue(localLibDirectory.exists());
assertTrue(localLibDirectory.isDirectory());
assertEquals(2, Stream.of(localLibDirectory.listFiles()).filter(file -> file.getName().endsWith(".dat")).count());
[6/8] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-1223'
Posted by ok...@apache.org.
Merge branch 'TINKERPOP-1223'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ea30b1c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ea30b1c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ea30b1c1
Branch: refs/heads/master
Commit: ea30b1c15dcd5f8e50577eb0efaaab97cea5cd7f
Parents: ce98e43 bef52c6
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 29 13:00:45 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 29 13:00:45 2016 -0600
----------------------------------------------------------------------
.../process/computer/GiraphGraphComputer.java | 2 +-
.../computer/AbstractHadoopGraphComputer.java | 30 +++++++
.../AbstractHadoopGraphComputerTest.java | 83 ++++++++++++++++++++
.../process/computer/SparkGraphComputer.java | 19 +++--
4 files changed, 126 insertions(+), 8 deletions(-)
----------------------------------------------------------------------