You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/08/19 16:28:00 UTC
incubator-tinkerpop git commit: Unified dependency grabbing for both
console and server.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/tp30 7cbcaf358 -> 462fe0a26
Unified dependency grabbing for both console and server.
Now that this code is the same between console and server there shouldn't be divergence in how it :install works. This change also fixes Gremlin Console on windows given TINKERPOP3-804
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/462fe0a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/462fe0a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/462fe0a2
Branch: refs/heads/tp30
Commit: 462fe0a26b8d81f636800392b6d9ee78ad96c8f0
Parents: 7cbcaf3
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Aug 19 10:26:30 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Aug 19 10:26:30 2015 -0400
----------------------------------------------------------------------
.../console/commands/InstallCommand.groovy | 160 +------------------
.../groovy/util/DependencyGrabber.groovy | 40 ++++-
2 files changed, 39 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/462fe0a2/gremlin-console/src/main/groovy/org/apache/tinkerpop/gremlin/console/commands/InstallCommand.groovy
----------------------------------------------------------------------
diff --git a/gremlin-console/src/main/groovy/org/apache/tinkerpop/gremlin/console/commands/InstallCommand.groovy b/gremlin-console/src/main/groovy/org/apache/tinkerpop/gremlin/console/commands/InstallCommand.groovy
index 1862515..6489475 100644
--- a/gremlin-console/src/main/groovy/org/apache/tinkerpop/gremlin/console/commands/InstallCommand.groovy
+++ b/gremlin-console/src/main/groovy/org/apache/tinkerpop/gremlin/console/commands/InstallCommand.groovy
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.console.plugin.PluggedIn
import org.apache.tinkerpop.gremlin.groovy.plugin.Artifact
import org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin
import groovy.grape.Grape
+import org.apache.tinkerpop.gremlin.groovy.util.DependencyGrabber
import org.codehaus.groovy.tools.shell.CommandSupport
import org.codehaus.groovy.tools.shell.Groovysh
@@ -48,141 +49,15 @@ class InstallCommand extends CommandSupport {
@Override
def Object execute(final List<String> arguments) {
final def artifact = createArtifact(arguments)
- final def dep = makeDepsMap(artifact)
- final def pluginsThatNeedRestart = grabDeps(dep)
-
- final String extClassPath = getPathFromDependency(dep)
- final String extLibPath = extClassPath + fileSep + "lib"
- final String extPluginPath = extClassPath + fileSep + "plugin"
- final File f = new File(extClassPath)
- if (f.exists())
- return "A module with the name ${dep.module} is already installed"
- else {
- f.mkdirs()
- new File(extLibPath).mkdirs()
- new File(extPluginPath).mkdirs()
- new File(extClassPath + fileSep + "plugin-info.txt").withWriter { out -> out << arguments.join(":") }
- }
-
- final def dependencyLocations = Grape.resolve([classLoader: shell.getInterp().getClassLoader()], null, dep)
-
- def fs = FileSystems.default
- def targetPluginPath = fs.getPath(extPluginPath)
- def targetLibPath = fs.getPath(extLibPath)
-
- // collect the files already on the path in /lib. making some unfortunate assumptions about what the path
- // looks like for the gremlin distribution
- def filesAlreadyInPath = []
- def libClassPath
- try {
- libClassPath = fs.getPath(System.getProperty("user.dir") + fileSep + "lib")
- getFileNames(filesAlreadyInPath, libClassPath)
- } catch (Exception ignored) {
- // the user might have a non-standard directory system. if they are non-standard then they must be
- // smart and they are therefore capable of resolving their own dependency problems. this could also
- // mean that they are running gremlin from source and not from target/*standalone*
- io.println "Detected a non-standard Gremlin directory structure during install. Expecting a 'lib' " +
- "directory sibling to 'ext'. This message does not necessarily imply failure, however " +
- "the console requires a certain directory structure for proper execution. Altering that " +
- "structure can lead to unexpected behavior."
- }
-
- // for the "plugin" path ignore slf4j related jars. they are already in the path and will create duplicate
- // bindings which generate annoying log messages that make you think stuff is wrong. also, don't bring
- // over files that are already on the path
- dependencyLocations.collect { fs.getPath(it.path) }
- .findAll { !(it.fileName.toFile().name ==~ /(slf4j|logback\-classic)-.*\.jar/) }
- .findAll { !filesAlreadyInPath.collect { it.getFileName().toString() }.contains(it.fileName.toFile().name) }
- .each { Files.copy(it, targetPluginPath.resolve(it.fileName), StandardCopyOption.REPLACE_EXISTING) }
-
- // for the "lib" path include all dependencies
- dependencyLocations.collect { fs.getPath(it.path) }
- .each { Files.copy(it, targetLibPath.resolve(it.fileName), StandardCopyOption.REPLACE_EXISTING) }
-
- // additional dependencies are outside those pulled by grape and are defined in the manifest of the plugin jar.
- // if a plugin uses that setting, it should force "restart" when the plugin is activated. right now,
- // it is up to the plugin developer to enforce that setting.
- getAdditionalDependencies(targetPluginPath, artifact).collect { fs.getPath(it.path) }
- .findAll { !(it.fileName.toFile().name ==~ /(slf4j|logback\-classic)-.*\.jar/) }
- .findAll { !filesAlreadyInPath.collect { it.getFileName().toString() }.contains(it.fileName.toFile().name)}
- .each { Files.copy(it, targetPluginPath.resolve(it.fileName), StandardCopyOption.REPLACE_EXISTING) }
-
- // for the "lib" path include all dependencies
- getAdditionalDependencies(targetLibPath, artifact).collect { fs.getPath(it.path) }
- .each { Files.copy(it, targetLibPath.resolve(it.fileName), StandardCopyOption.REPLACE_EXISTING) }
-
- // the ordering of jars seems to matter in some cases (e.g. neo4j). the plugin system allows the plugin
- // to place a Gremlin-Plugin-Paths entry in the jar manifest file to define where specific jar files should
- // go in the path which provides enough flexibility to control when jars should load. unfortunately,
- // this "ordering" issue doesn't seem to be documented as an issue anywhere and it is difficult to say
- // whether it is a java issue, groovy classloader issue, grape issue, etc. see this issue for more
- // on the weirdness: https://github.org/apache/tinkerpop/tinkerpop3/issues/230
- //
- // another unfortunate side-effect to this approach is that manual cleanup of jars is kinda messy now
- // because you can't just delete the plugin directory as one or more of the jars might have been moved.
- // unsure of what the long term effects of this is. at the end of the day, users may simply need to
- // know something about their dependencies in order to have lots of "installed" plugins/dependencies.
- alterPaths("Gremlin-Plugin-Paths", targetPluginPath, artifact)
- alterPaths("Gremlin-Lib-Paths", targetLibPath, artifact)
-
- return "Loaded: " + arguments + (pluginsThatNeedRestart.size() == 0 ? "" : " - restart the console to use $pluginsThatNeedRestart")
- }
-
- private static String getPathFromDependency(final Map<String, Object> dep) {
- return System.getProperty("user.dir") + fileSep + "ext" + fileSep + (String) dep.module
- }
-
- private static alterPaths(final String manifestEntry, final Path extPath, final Artifact artifact) {
- try {
- // another assumption about the pathing - seems safe for right now as the :install command is
- // responsible for all this stuff. if the user chooses to manually install their dependencies
- // to the console, then it's up to them to sort this stuff out.
- def pathToInstalled = extPath.resolve(artifact.artifact + "-" + artifact.version + ".jar")
- final JarFile jar = new JarFile(pathToInstalled.toFile());
- final Manifest manifest = jar.getManifest()
-
- // containsKey doesn't seem to want to work - so just check for null - dah
- def attrLine = manifest.mainAttributes.getValue(manifestEntry)
- if (attrLine != null) {
- def splitLine = attrLine.split(";")
- splitLine.each {
- if (it.endsWith("="))
- Files.delete(extPath.resolve(it.substring(0, it.length() - 1)))
- else {
- def kv = it.split("=")
- Files.move(extPath.resolve(kv[0]), extPath.resolve(kv[1]), StandardCopyOption.REPLACE_EXISTING)
- }
- }
- }
- } catch (Exception ex) {
- // errors here will likely have to do with bad pathing or poorly constructed entries in the manifest.
- // hopefully these will only occur for developers of plugins who need to make use of this function.
- // internally to tinkerpop this is a neo4j-only issue
- throw new RuntimeException(ex)
- }
- }
-
- private Set<URI> getAdditionalDependencies(final Path extPath, final Artifact artifact) {
try {
- def pathToInstalled = extPath.resolve(artifact.artifact + "-" + artifact.version + ".jar")
- final JarFile jar = new JarFile(pathToInstalled.toFile())
- final Manifest manifest = jar.getManifest()
- def attrLine = manifest.mainAttributes.getValue("Gremlin-Plugin-Dependencies")
- def additionalDependencies = [] as Set<URI>
- if (attrLine != null) {
- def splitLine = attrLine.split(";")
- splitLine.each {
- def artifactBits = it.split(":")
- def additional = new Artifact(artifactBits[0], artifactBits[1], artifactBits[2])
-
- final def additionalDep = makeDepsMap(additional)
- additionalDependencies.addAll(Grape.resolve([classLoader: shell.getInterp().getClassLoader()], null, additionalDep))
- }
- }
+ def grabber = new DependencyGrabber(shell.getInterp().getClassLoader(), System.getProperty("user.dir") + fileSep + "ext")
+ grabber.copyDependenciesToPath(artifact)
- return additionalDependencies
+ final def dep = grabber.makeDepsMap(artifact)
+ final def pluginsThatNeedRestart = grabDeps(dep)
+ return "Loaded: " + arguments + (pluginsThatNeedRestart.size() == 0 ? "" : " - restart the console to use $pluginsThatNeedRestart")
} catch (Exception ex) {
- throw new RuntimeException(ex)
+ return ex.message
}
}
@@ -220,25 +95,4 @@ class InstallCommand extends CommandSupport {
return new Artifact(group, module, version)
}
-
- private def makeDepsMap(final Artifact artifact) {
- final Map<String, Object> map = new HashMap<>()
- map.put("classLoader", shell.getInterp().getClassLoader())
- map.put("group", artifact.getGroup())
- map.put("module", artifact.getArtifact())
- map.put("version", artifact.getVersion())
- map.put("changing", false)
- return map
- }
-
- private static void getFileNames(final List fileNames, final Path dir) {
- final DirectoryStream<Path> stream = Files.newDirectoryStream(dir)
- for (Path path : stream) {
- if (path.toFile().isDirectory()) getFileNames(fileNames, path)
- else {
- fileNames.add(path.toAbsolutePath())
- }
- }
- stream.close()
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/462fe0a2/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/util/DependencyGrabber.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/util/DependencyGrabber.groovy b/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/util/DependencyGrabber.groovy
index 2e8877d..01b5074 100644
--- a/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/util/DependencyGrabber.groovy
+++ b/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/util/DependencyGrabber.groovy
@@ -21,6 +21,8 @@ package org.apache.tinkerpop.gremlin.groovy.util
import groovy.grape.Grape
import org.apache.commons.lang3.SystemUtils
import org.apache.tinkerpop.gremlin.groovy.plugin.Artifact
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
import java.nio.file.*
import java.util.jar.JarFile
@@ -33,6 +35,7 @@ import java.util.jar.Manifest
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
class DependencyGrabber {
+ private static final Logger logger = LoggerFactory.getLogger(DependencyGrabber.class);
private final static String fileSep = System.getProperty("file.separator")
private final ClassLoader classLoaderToUse
@@ -43,7 +46,7 @@ class DependencyGrabber {
this.extensionDirectory = extensionDirectory
}
- def void copyDependenciesToPath(final Artifact artifact) {
+ def Set<String> copyDependenciesToPath(final Artifact artifact) {
final def dep = makeDepsMap(artifact)
final String extClassPath = getPathFromDependency(dep)
final String extLibPath = extClassPath + fileSep + "lib"
@@ -61,22 +64,33 @@ class DependencyGrabber {
def targetPluginPath = fs.getPath(extPluginPath)
def targetLibPath = fs.getPath(extLibPath)
+ // collect the files already on the path in /lib. making some unfortunate assumptions about what the path
+ // looks like for the gremlin distribution
def filesAlreadyInPath = []
def libClassPath
try {
libClassPath = fs.getPath(System.getProperty("user.dir") + fileSep + "lib")
getFileNames(filesAlreadyInPath, libClassPath)
} catch (Exception ignored) {
- println "Detected a non-standard Gremlin directory structure during install. Expecting a 'lib' " +
+ // the user might have a non-standard directory system. if they are non-standard then they must be
+ // smart and they are therefore capable of resolving their own dependency problems. this could also
+ // mean that they are running gremlin from source and not from target/*standalone*
+ logger.warn("Detected a non-standard Gremlin directory structure during install. Expecting a 'lib' " +
"directory sibling to 'ext'. This message does not necessarily imply failure, however " +
"the console requires a certain directory structure for proper execution. Altering that " +
- "structure can lead to unexpected behavior."
+ "structure can lead to unexpected behavior.")
}
final def dependencyLocations = [] as Set<URI>
dependencyLocations.addAll(Grape.resolve([classLoader: this.classLoaderToUse], null, dep))
- // get dependencies for the plugin path which should be part of the class path
+ // for the "plugin" path ignore slf4j related jars. they are already in the path and will create duplicate
+ // bindings which generate annoying log messages that make you think stuff is wrong. also, don't bring
+ // over files that are already on the path. these dependencies will be part of the classpath
+ //
+ // additional dependencies are outside those pulled by grape and are defined in the manifest of the plugin jar.
+ // if a plugin uses that setting, it should force "restart" when the plugin is activated. right now,
+ // it is up to the plugin developer to enforce that setting.
dependencyLocations.collect(convertUriToPath(fs))
.findAll { !(it.fileName.toFile().name ==~ /(slf4j|logback\-classic)-.*\.jar/) }
.findAll {!filesAlreadyInPath.collect { it.getFileName().toString() }.contains(it.fileName.toFile().name)}
@@ -90,6 +104,17 @@ class DependencyGrabber {
dependencyLocations.collect(convertUriToPath(fs)).each(copyTo(targetLibPath))
getAdditionalDependencies(targetLibPath, artifact).collect(convertUriToPath(fs)).each(copyTo(targetLibPath))
+ // the ordering of jars seems to matter in some cases (e.g. neo4j). the plugin system allows the plugin
+ // to place a Gremlin-Plugin-Paths entry in the jar manifest file to define where specific jar files should
+ // go in the path which provides enough flexibility to control when jars should load. unfortunately,
+ // this "ordering" issue doesn't seem to be documented as an issue anywhere and it is difficult to say
+ // whether it is a java issue, groovy classloader issue, grape issue, etc. see this issue for more
+ // on the weirdness: https://github.org/apache/tinkerpop/tinkerpop3/issues/230
+ //
+ // another unfortunate side-effect to this approach is that manual cleanup of jars is kinda messy now
+ // because you can't just delete the plugin directory as one or more of the jars might have been moved.
+ // unsure of what the long term effects of this is. at the end of the day, users may simply need to
+ // know something about their dependencies in order to have lots of "installed" plugins/dependencies.
alterPaths("Gremlin-Plugin-Paths", targetPluginPath, artifact)
alterPaths("Gremlin-Lib-Paths", targetLibPath, artifact)
}
@@ -100,7 +125,7 @@ class DependencyGrabber {
def copying = path.resolve(p.fileName)
if (!copying.toFile().exists()) {
Files.copy(p, copying, StandardCopyOption.REPLACE_EXISTING)
- println "Copying - $copying"
+ logger.info("Copying - $copying")
}
}
}
@@ -163,11 +188,10 @@ class DependencyGrabber {
}
private String getPathFromDependency(final Map<String, Object> dep) {
- def fileSep = System.getProperty("file.separator")
return this.extensionDirectory + fileSep + (String) dep.module
}
- private def makeDepsMap(final Artifact artifact) {
+ public def makeDepsMap(final Artifact artifact) {
final Map<String, Object> map = new HashMap<>()
map.put("classLoader", this.classLoaderToUse)
map.put("group", artifact.getGroup())
@@ -177,7 +201,7 @@ class DependencyGrabber {
return map
}
- private static void getFileNames(final List fileNames, final Path dir) {
+ public static void getFileNames(final List fileNames, final Path dir) {
final DirectoryStream<Path> stream = Files.newDirectoryStream(dir)
for (Path path : stream) {
if (path.toFile().isDirectory()) getFileNames(fileNames, path)