You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/03/29 21:01:44 UTC

[1/8] incubator-tinkerpop git commit: If the HADOOP_GREMLIN_LIBS path does not exist locally, then search for it in HDFS. If it exists there, copy it to a java.io.tmp directory and then pull out the jars from there to upload to the DistributdCache (Girap

Repository: incubator-tinkerpop
Updated Branches:
  refs/heads/master 8a1d8a1e0 -> b78ffcf3c


If the HADOOP_GREMLIN_LIBS path does not exist locally, then search for it in HDFS. If it exists there, copy it to a java.io.tmp directory and then pull out the jars from there to upload to the DistributdCache (Giraph) or SparkServer (Spark).


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/c6b9bc71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/c6b9bc71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/c6b9bc71

Branch: refs/heads/master
Commit: c6b9bc7114aa77b69edf942486517c04794c7d45
Parents: da8c5fb
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Mar 28 14:18:01 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Mar 28 14:18:01 2016 -0600

----------------------------------------------------------------------
 .../process/computer/GiraphGraphComputer.java   |  2 +-
 .../computer/AbstractHadoopGraphComputer.java   | 30 +++++++
 .../AbstractHadoopGraphComputerTest.java        | 82 ++++++++++++++++++++
 .../process/computer/SparkGraphComputer.java    | 19 +++--
 4 files changed, 125 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c6b9bc71/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
index 76eda1d..3ed8618 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
@@ -247,7 +247,7 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
             else {
                 final String[] paths = hadoopGremlinLibsLocal.split(":");
                 for (final String path : paths) {
-                    final File file = new File(path);
+                    final File file = AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, path);
                     if (file.exists()) {
                         Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> {
                             try {

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c6b9bc71/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
index b766089..7659f0d 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
@@ -18,6 +18,10 @@
  */
 package org.apache.tinkerpop.gremlin.hadoop.process.computer;
 
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.tinkerpop.gremlin.hadoop.Constants;
 import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
@@ -31,9 +35,12 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+import org.apache.tinkerpop.gremlin.util.Gremlin;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.IOException;
 import java.util.HashSet;
 import java.util.Optional;
 import java.util.Set;
@@ -195,4 +202,27 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
             return false;
         }
     }
+
+    //////////
+
+    public static File copyDirectoryIfNonExistent(final FileSystem fileSystem, final String localDirectory) {
+        try {
+            final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
+            File file = new File(localDirectory);
+            if ((Boolean.valueOf(System.getProperty("is.testing", "false")) || !file.exists()) && fileSystem.exists(new Path(localDirectory)) && fileSystem.isDirectory(new Path(localDirectory))) {
+                final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
+                if (!tempDirectory.exists()) assert tempDirectory.mkdir();
+                final String tempPath = tempDirectory.getAbsolutePath() + "/" + new File(localDirectory).getName();
+                final RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path(localDirectory), false);
+                while (files.hasNext()) {
+                    final LocatedFileStatus f = files.next();
+                    fileSystem.copyToLocalFile(f.getPath(), new Path(tempPath + "/" + f.getPath().getName()));
+                }
+                return new File(tempPath);
+            } else
+                return file;
+        } catch (final IOException e) {
+            throw new IllegalStateException(e.getMessage(), e);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c6b9bc71/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
new file mode 100644
index 0000000..99916db
--- /dev/null
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.hadoop.process.computer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tinkerpop.gremlin.util.Gremlin;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class AbstractHadoopGraphComputerTest {
+
+    @Test
+    public void shouldCopyDirectoriesCorrectly() throws Exception {
+        final FileSystem fs = FileSystem.get(new Configuration());
+        if (!new File(System.getProperty("java.io.tmpdir") + "/tmp").exists())
+            assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").mkdir());
+        File tempFile1 = new File(System.getProperty("java.io.tmpdir") + "/tmp/test1.txt");
+        File tempFile2 = new File(System.getProperty("java.io.tmpdir") + "/tmp/test2.txt");
+        assertTrue(tempFile1.createNewFile());
+        assertTrue(tempFile2.createNewFile());
+        assertTrue(tempFile1.exists());
+        assertTrue(tempFile2.exists());
+        if (fs.exists(new Path("target/testing")))
+            assertTrue(fs.delete(new Path("target/testing"), true));
+        fs.copyFromLocalFile(true, new Path(tempFile1.getAbsolutePath()), new Path("target/testing/test1.dat"));
+        fs.copyFromLocalFile(true, new Path(tempFile2.getAbsolutePath()), new Path("target/testing/test2.dat"));
+        assertTrue(fs.exists(new Path("target/testing/test1.dat")));
+        assertTrue(fs.exists(new Path("target/testing/test2.dat")));
+        assertTrue(fs.exists(new Path("target/testing")));
+        assertTrue(fs.isDirectory(new Path("target/testing")));
+        assertFalse(tempFile1.exists());
+        assertFalse(tempFile2.exists());
+        assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").exists());
+        assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").delete());
+        assertTrue(fs.exists(new Path("target/testing/test1.dat")));
+        assertTrue(fs.exists(new Path("target/testing/test2.dat")));
+        assertTrue(fs.exists(new Path("target/testing")));
+        assertTrue(fs.isDirectory(new Path("target/testing")));
+        /////
+        final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
+        final File localDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
+        final File localLibDirectory = new File(localDirectory.getAbsolutePath() + "/testing");
+        assertTrue(localDirectory.exists());
+        if (localLibDirectory.exists()) {
+            Stream.of(localLibDirectory.listFiles()).forEach(File::delete);
+            assertTrue(localLibDirectory.delete());
+        }
+        assertFalse(localLibDirectory.exists());
+        assertEquals(localLibDirectory, AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, "target/testing"));
+        assertTrue(localLibDirectory.exists());
+        assertTrue(localLibDirectory.isDirectory());
+        assertEquals(2, Stream.of(localLibDirectory.listFiles()).filter(file -> file.getName().endsWith(".dat")).count());
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c6b9bc71/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index f3b2eb9..928a880 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -323,13 +323,18 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
             if (null == hadoopGremlinLocalLibs)
                 this.logger.warn(Constants.HADOOP_GREMLIN_LIBS + " is not set -- proceeding regardless");
             else {
-                final String[] paths = hadoopGremlinLocalLibs.split(":");
-                for (final String path : paths) {
-                    final File file = new File(path);
-                    if (file.exists())
-                        Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> sparkContext.addJar(f.getAbsolutePath()));
-                    else
-                        this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
+                try {
+                    final String[] paths = hadoopGremlinLocalLibs.split(":");
+                    final FileSystem fs = FileSystem.get(hadoopConfiguration);
+                    for (final String path : paths) {
+                        final File file = AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, path);
+                        if (file.exists())
+                            Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> sparkContext.addJar(f.getAbsolutePath()));
+                        else
+                            this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
+                    }
+                } catch (IOException e) {
+                    throw new IllegalStateException(e.getMessage(), e);
                 }
             }
         }


[7/8] incubator-tinkerpop git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-tinkerpop

Posted by ok...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-tinkerpop


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/70b7fc35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/70b7fc35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/70b7fc35

Branch: refs/heads/master
Commit: 70b7fc35b1282fa262559e6c1c0ddb595d1d8bdd
Parents: ea30b1c 8a1d8a1
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 29 13:00:53 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 29 13:00:53 2016 -0600

----------------------------------------------------------------------
 .../src/reference/gremlin-applications.asciidoc |  3 ++
 .../gremlin/groovy/engine/GremlinExecutor.java  | 27 +++++++++++++++---
 .../groovy/engine/GremlinExecutorTest.java      | 29 ++++++++++++++++++++
 .../op/traversal/TraversalOpProcessor.java      | 12 +++++++-
 4 files changed, 66 insertions(+), 5 deletions(-)
----------------------------------------------------------------------



[3/8] incubator-tinkerpop git commit: made mkdir() to mkdirs() per @dkuppitz's request.

Posted by ok...@apache.org.
made mkdir() to mkdirs() per @dkuppitz's request.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/da791aa9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/da791aa9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/da791aa9

Branch: refs/heads/master
Commit: da791aa9b9c772050cf6c7797f28622b7663f1dc
Parents: cbd8200
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Mar 28 18:17:46 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Mar 28 18:17:46 2016 -0600

----------------------------------------------------------------------
 .../hadoop/process/computer/AbstractHadoopGraphComputer.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/da791aa9/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
index 7659f0d..28dd7e5 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
@@ -211,7 +211,7 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
             File file = new File(localDirectory);
             if ((Boolean.valueOf(System.getProperty("is.testing", "false")) || !file.exists()) && fileSystem.exists(new Path(localDirectory)) && fileSystem.isDirectory(new Path(localDirectory))) {
                 final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
-                if (!tempDirectory.exists()) assert tempDirectory.mkdir();
+                if (!tempDirectory.exists()) assert tempDirectory.mkdirs();
                 final String tempPath = tempDirectory.getAbsolutePath() + "/" + new File(localDirectory).getName();
                 final RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path(localDirectory), false);
                 while (files.hasNext()) {


[4/8] incubator-tinkerpop git commit: added System.getProperty(file.separator)

Posted by ok...@apache.org.
added System.getProperty(file.separator)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/79009e60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/79009e60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/79009e60

Branch: refs/heads/master
Commit: 79009e60d5f35cb4fc92801b23ccc1edec8662cf
Parents: da791aa
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 29 08:46:42 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 29 08:46:42 2016 -0600

----------------------------------------------------------------------
 .../hadoop/process/computer/AbstractHadoopGraphComputer.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/79009e60/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
index 28dd7e5..a05a1be 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
@@ -210,13 +210,13 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
             final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
             File file = new File(localDirectory);
             if ((Boolean.valueOf(System.getProperty("is.testing", "false")) || !file.exists()) && fileSystem.exists(new Path(localDirectory)) && fileSystem.isDirectory(new Path(localDirectory))) {
-                final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
+                final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + hadoopGremlinLibsRemote);
                 if (!tempDirectory.exists()) assert tempDirectory.mkdirs();
-                final String tempPath = tempDirectory.getAbsolutePath() + "/" + new File(localDirectory).getName();
+                final String tempPath = tempDirectory.getAbsolutePath() + System.getProperty("file.separator") + new File(localDirectory).getName();
                 final RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path(localDirectory), false);
                 while (files.hasNext()) {
                     final LocatedFileStatus f = files.next();
-                    fileSystem.copyToLocalFile(f.getPath(), new Path(tempPath + "/" + f.getPath().getName()));
+                    fileSystem.copyToLocalFile(f.getPath(), new Path(tempPath + System.getProperty("file.separator") + f.getPath().getName()));
                 }
                 return new File(tempPath);
             } else


[2/8] incubator-tinkerpop git commit: dar, test was bad. had residual directories that were not cleaned up. Thus, on a fresh system, it failed.

Posted by ok...@apache.org.
dar, test was bad. had residual directories that were not cleaned up. Thus, on a fresh system, it failed.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/cbd8200a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/cbd8200a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/cbd8200a

Branch: refs/heads/master
Commit: cbd8200aa26ced615ecd6c1d8eb007ea3417b634
Parents: c6b9bc7
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Mar 28 15:00:24 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Mar 28 15:00:24 2016 -0600

----------------------------------------------------------------------
 .../hadoop/process/computer/AbstractHadoopGraphComputerTest.java    | 1 -
 1 file changed, 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/cbd8200a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
index 99916db..4a2c8a9 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
@@ -68,7 +68,6 @@ public class AbstractHadoopGraphComputerTest {
         final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
         final File localDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
         final File localLibDirectory = new File(localDirectory.getAbsolutePath() + "/testing");
-        assertTrue(localDirectory.exists());
         if (localLibDirectory.exists()) {
             Stream.of(localLibDirectory.listFiles()).forEach(File::delete);
             assertTrue(localLibDirectory.delete());


[8/8] incubator-tinkerpop git commit: updated CHANGELOG.

Posted by ok...@apache.org.
updated CHANGELOG.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b78ffcf3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b78ffcf3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b78ffcf3

Branch: refs/heads/master
Commit: b78ffcf3c47ae135abb23258e6bc465393dd06ab
Parents: 70b7fc3
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 29 13:01:35 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 29 13:01:35 2016 -0600

----------------------------------------------------------------------
 CHANGELOG.asciidoc | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b78ffcf3/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 87a7ba6..597e27d 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
 TinkerPop 3.2.0 (NOT OFFICIALLY RELEASED YET)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+* `HADOOP_GREMLIN_LIBS` can now reference a directory in HDFS and will be used if the directory does not exist locally.
 * Added `gremlin-benchmark` module with JMH benchmarking base classes that can be used for further benchmark development.
 * `TraversalStrategies.GlobalCache` supports both `Graph` and `GraphComputer` strategy registrations.
 * `select("a","b").by("name").by("age")`-style traversals now work in OLAP with new `PathProcessorStrategy`.


[5/8] incubator-tinkerpop git commit: used testclassname-based directories to reduce the probability of target/ directory conflicts.

Posted by ok...@apache.org.
used testclassname-based directories to reduce the probability of target/ directory conflicts.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/bef52c61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/bef52c61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/bef52c61

Branch: refs/heads/master
Commit: bef52c611fa1484774df02d049d03bb06f7d07f4
Parents: 79009e6
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 29 08:51:42 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 29 08:51:42 2016 -0600

----------------------------------------------------------------------
 .../AbstractHadoopGraphComputerTest.java        | 42 ++++++++++----------
 1 file changed, 22 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bef52c61/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
index 4a2c8a9..56f0417 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputerTest.java
@@ -39,41 +39,43 @@ public class AbstractHadoopGraphComputerTest {
 
     @Test
     public void shouldCopyDirectoriesCorrectly() throws Exception {
+        final String hdfsName = this.getClass().getSimpleName() + "-hdfs";
+        final String localName = this.getClass().getSimpleName() + "-local";
         final FileSystem fs = FileSystem.get(new Configuration());
-        if (!new File(System.getProperty("java.io.tmpdir") + "/tmp").exists())
-            assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").mkdir());
-        File tempFile1 = new File(System.getProperty("java.io.tmpdir") + "/tmp/test1.txt");
-        File tempFile2 = new File(System.getProperty("java.io.tmpdir") + "/tmp/test2.txt");
+        if (!new File(System.getProperty("java.io.tmpdir") + "/" + localName).exists())
+            assertTrue(new File(System.getProperty("java.io.tmpdir") + "/" + localName).mkdir());
+        File tempFile1 = new File(System.getProperty("java.io.tmpdir") + "/" + localName + "/test1.txt");
+        File tempFile2 = new File(System.getProperty("java.io.tmpdir") + "/" + localName + "/test2.txt");
         assertTrue(tempFile1.createNewFile());
         assertTrue(tempFile2.createNewFile());
         assertTrue(tempFile1.exists());
         assertTrue(tempFile2.exists());
-        if (fs.exists(new Path("target/testing")))
-            assertTrue(fs.delete(new Path("target/testing"), true));
-        fs.copyFromLocalFile(true, new Path(tempFile1.getAbsolutePath()), new Path("target/testing/test1.dat"));
-        fs.copyFromLocalFile(true, new Path(tempFile2.getAbsolutePath()), new Path("target/testing/test2.dat"));
-        assertTrue(fs.exists(new Path("target/testing/test1.dat")));
-        assertTrue(fs.exists(new Path("target/testing/test2.dat")));
-        assertTrue(fs.exists(new Path("target/testing")));
-        assertTrue(fs.isDirectory(new Path("target/testing")));
+        if (fs.exists(new Path("target/" + hdfsName)))
+            assertTrue(fs.delete(new Path("target/" + hdfsName), true));
+        fs.copyFromLocalFile(true, new Path(tempFile1.getAbsolutePath()), new Path("target/" + hdfsName + "/test1.dat"));
+        fs.copyFromLocalFile(true, new Path(tempFile2.getAbsolutePath()), new Path("target/" + hdfsName + "/test2.dat"));
+        assertTrue(fs.exists(new Path("target/" + hdfsName + "/test1.dat")));
+        assertTrue(fs.exists(new Path("target/" + hdfsName + "/test2.dat")));
+        assertTrue(fs.exists(new Path("target/" + hdfsName)));
+        assertTrue(fs.isDirectory(new Path("target/" + hdfsName)));
         assertFalse(tempFile1.exists());
         assertFalse(tempFile2.exists());
-        assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").exists());
-        assertTrue(new File(System.getProperty("java.io.tmpdir") + "/tmp").delete());
-        assertTrue(fs.exists(new Path("target/testing/test1.dat")));
-        assertTrue(fs.exists(new Path("target/testing/test2.dat")));
-        assertTrue(fs.exists(new Path("target/testing")));
-        assertTrue(fs.isDirectory(new Path("target/testing")));
+        assertTrue(new File(System.getProperty("java.io.tmpdir") + "/" + localName).exists());
+        assertTrue(new File(System.getProperty("java.io.tmpdir") + "/" + localName).delete());
+        assertTrue(fs.exists(new Path("target/" + hdfsName + "/test1.dat")));
+        assertTrue(fs.exists(new Path("target/" + hdfsName + "/test2.dat")));
+        assertTrue(fs.exists(new Path("target/" + hdfsName)));
+        assertTrue(fs.isDirectory(new Path("target/" + hdfsName)));
         /////
         final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
         final File localDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote);
-        final File localLibDirectory = new File(localDirectory.getAbsolutePath() + "/testing");
+        final File localLibDirectory = new File(localDirectory.getAbsolutePath() + "/" + hdfsName);
         if (localLibDirectory.exists()) {
             Stream.of(localLibDirectory.listFiles()).forEach(File::delete);
             assertTrue(localLibDirectory.delete());
         }
         assertFalse(localLibDirectory.exists());
-        assertEquals(localLibDirectory, AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, "target/testing"));
+        assertEquals(localLibDirectory, AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, "target/" + hdfsName));
         assertTrue(localLibDirectory.exists());
         assertTrue(localLibDirectory.isDirectory());
         assertEquals(2, Stream.of(localLibDirectory.listFiles()).filter(file -> file.getName().endsWith(".dat")).count());


[6/8] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-1223'

Posted by ok...@apache.org.
Merge branch 'TINKERPOP-1223'


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ea30b1c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ea30b1c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ea30b1c1

Branch: refs/heads/master
Commit: ea30b1c15dcd5f8e50577eb0efaaab97cea5cd7f
Parents: ce98e43 bef52c6
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 29 13:00:45 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 29 13:00:45 2016 -0600

----------------------------------------------------------------------
 .../process/computer/GiraphGraphComputer.java   |  2 +-
 .../computer/AbstractHadoopGraphComputer.java   | 30 +++++++
 .../AbstractHadoopGraphComputerTest.java        | 83 ++++++++++++++++++++
 .../process/computer/SparkGraphComputer.java    | 19 +++--
 4 files changed, 126 insertions(+), 8 deletions(-)
----------------------------------------------------------------------