You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2016/06/09 16:31:06 UTC
incubator-tinkerpop git commit: Minor tweaks and support for more
path separators in `HADOOP_GREMLIN_LIBS` (e.g. `; ` on Windows systems).
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1331 91475d3e8 -> 2ccae55e8
Minor tweaks and support for more path separators in `HADOOP_GREMLIN_LIBS` (e.g. `;` on Windows systems).
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/2ccae55e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/2ccae55e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/2ccae55e
Branch: refs/heads/TINKERPOP-1331
Commit: 2ccae55e893b92c1f58b7b878ffa5a27b4828afa
Parents: 91475d3
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Thu Jun 9 18:28:30 2016 +0200
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Thu Jun 9 18:28:30 2016 +0200
----------------------------------------------------------------------
.../computer/AbstractHadoopGraphComputer.java | 25 ++++++++++----------
1 file changed, 12 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2ccae55e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
index f5f332d..6a68046 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
@@ -49,14 +49,14 @@ import java.util.Optional;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Stream;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
public abstract class AbstractHadoopGraphComputer implements GraphComputer {
- private final static Pattern PATH_PATTERN = Pattern.compile("([^:]|://)+");
+ private final static Pattern PATH_PATTERN =
+ Pattern.compile(File.pathSeparator.equals(":") ? "([^:]|://)+" : ("[^" + File.pathSeparator + "]"));
protected final Logger logger;
protected final HadoopGraph hadoopGraph;
@@ -171,8 +171,7 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
loadJar(hadoopConfiguration, f, params);
}
}
- }
- else
+ } else
this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
}
} catch (IOException e) {
@@ -251,22 +250,22 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
//////////
- public static File copyDirectoryIfNonExistent(final FileSystem fileSystem, final String localDirectory) {
+ public static File copyDirectoryIfNonExistent(final FileSystem fileSystem, final String directory) {
try {
final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs";
- File file = new File(localDirectory);
- if ((Boolean.valueOf(System.getProperty("is.testing", "false")) || !file.exists()) && fileSystem.exists(new Path(localDirectory)) && fileSystem.isDirectory(new Path(localDirectory))) {
- final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + hadoopGremlinLibsRemote);
- if (!tempDirectory.exists()) assert tempDirectory.mkdirs();
- final String tempPath = tempDirectory.getAbsolutePath() + System.getProperty("file.separator") + new File(localDirectory).getName();
- final RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path(localDirectory), false);
+ final Path path = new Path(directory);
+ if (Boolean.valueOf(System.getProperty("is.testing", "false")) || (fileSystem.exists(path) && fileSystem.isDirectory(path))) {
+ final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + File.separator + hadoopGremlinLibsRemote);
+ assert tempDirectory.exists() || tempDirectory.mkdirs();
+ final String tempPath = tempDirectory.getAbsolutePath() + File.separator + path.getName();
+ final RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(path, false);
while (files.hasNext()) {
final LocatedFileStatus f = files.next();
- fileSystem.copyToLocalFile(f.getPath(), new Path(tempPath + System.getProperty("file.separator") + f.getPath().getName()));
+ fileSystem.copyToLocalFile(false, f.getPath(), new Path(tempPath + System.getProperty("file.separator") + f.getPath().getName()), true);
}
return new File(tempPath);
} else
- return file;
+ return new File(directory);
} catch (final IOException e) {
throw new IllegalStateException(e.getMessage(), e);
}