You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/08/19 16:28:00 UTC

incubator-tinkerpop git commit: Unified dependency grabbing for both console and server.

Repository: incubator-tinkerpop
Updated Branches:
  refs/heads/tp30 7cbcaf358 -> 462fe0a26


Unified dependency grabbing for both console and server.

Now that this code is the same between console and server there shouldn't be divergence in how it :install works.  This change also fixes Gremlin Console on windows given TINKERPOP3-804


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/462fe0a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/462fe0a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/462fe0a2

Branch: refs/heads/tp30
Commit: 462fe0a26b8d81f636800392b6d9ee78ad96c8f0
Parents: 7cbcaf3
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Aug 19 10:26:30 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Aug 19 10:26:30 2015 -0400

----------------------------------------------------------------------
 .../console/commands/InstallCommand.groovy      | 160 +------------------
 .../groovy/util/DependencyGrabber.groovy        |  40 ++++-
 2 files changed, 39 insertions(+), 161 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/462fe0a2/gremlin-console/src/main/groovy/org/apache/tinkerpop/gremlin/console/commands/InstallCommand.groovy
----------------------------------------------------------------------
diff --git a/gremlin-console/src/main/groovy/org/apache/tinkerpop/gremlin/console/commands/InstallCommand.groovy b/gremlin-console/src/main/groovy/org/apache/tinkerpop/gremlin/console/commands/InstallCommand.groovy
index 1862515..6489475 100644
--- a/gremlin-console/src/main/groovy/org/apache/tinkerpop/gremlin/console/commands/InstallCommand.groovy
+++ b/gremlin-console/src/main/groovy/org/apache/tinkerpop/gremlin/console/commands/InstallCommand.groovy
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.console.plugin.PluggedIn
 import org.apache.tinkerpop.gremlin.groovy.plugin.Artifact
 import org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin
 import groovy.grape.Grape
+import org.apache.tinkerpop.gremlin.groovy.util.DependencyGrabber
 import org.codehaus.groovy.tools.shell.CommandSupport
 import org.codehaus.groovy.tools.shell.Groovysh
 
@@ -48,141 +49,15 @@ class InstallCommand extends CommandSupport {
     @Override
     def Object execute(final List<String> arguments) {
         final def artifact = createArtifact(arguments)
-        final def dep = makeDepsMap(artifact)
-        final def pluginsThatNeedRestart = grabDeps(dep)
-
-        final String extClassPath = getPathFromDependency(dep)
-        final String extLibPath = extClassPath + fileSep + "lib"
-        final String extPluginPath = extClassPath + fileSep + "plugin"
-        final File f = new File(extClassPath)
-        if (f.exists())
-            return "A module with the name ${dep.module} is already installed"
-        else {
-            f.mkdirs()
-            new File(extLibPath).mkdirs()
-            new File(extPluginPath).mkdirs()
-            new File(extClassPath + fileSep + "plugin-info.txt").withWriter { out -> out << arguments.join(":") }
-        }
-
-        final def dependencyLocations = Grape.resolve([classLoader: shell.getInterp().getClassLoader()], null, dep)
-
-        def fs = FileSystems.default
-        def targetPluginPath = fs.getPath(extPluginPath)
-        def targetLibPath = fs.getPath(extLibPath)
-
-        // collect the files already on the path in /lib. making some unfortunate assumptions about what the path
-        // looks like for the gremlin distribution
-        def filesAlreadyInPath = []
-        def libClassPath
-        try {
-            libClassPath = fs.getPath(System.getProperty("user.dir") + fileSep + "lib")
-            getFileNames(filesAlreadyInPath, libClassPath)
-        } catch (Exception ignored) {
-            // the user might have a non-standard directory system.  if they are non-standard then they must be
-            // smart and they are therefore capable of resolving their own dependency problems.  this could also
-            // mean that they are running gremlin from source and not from target/*standalone*
-            io.println "Detected a non-standard Gremlin directory structure during install.  Expecting a 'lib' " +
-                    "directory sibling to 'ext'. This message does not necessarily imply failure, however " +
-                    "the console requires a certain directory structure for proper execution. Altering that " +
-                    "structure can lead to unexpected behavior."
-        }
-
-        // for the "plugin" path ignore slf4j related jars.  they are already in the path and will create duplicate
-        // bindings which generate annoying log messages that make you think stuff is wrong.  also, don't bring
-        // over files that are already on the path
-        dependencyLocations.collect { fs.getPath(it.path) }
-                .findAll { !(it.fileName.toFile().name ==~ /(slf4j|logback\-classic)-.*\.jar/) }
-                .findAll { !filesAlreadyInPath.collect { it.getFileName().toString() }.contains(it.fileName.toFile().name) }
-                .each { Files.copy(it, targetPluginPath.resolve(it.fileName), StandardCopyOption.REPLACE_EXISTING) }
-
-        // for the "lib" path include all dependencies
-        dependencyLocations.collect { fs.getPath(it.path) }
-                .each { Files.copy(it, targetLibPath.resolve(it.fileName), StandardCopyOption.REPLACE_EXISTING) }
-
-        // additional dependencies are outside those pulled by grape and are defined in the manifest of the plugin jar.
-        // if a plugin uses that setting, it should force "restart" when the plugin is activated.  right now,
-        // it is up to the plugin developer to enforce that setting.
-        getAdditionalDependencies(targetPluginPath, artifact).collect { fs.getPath(it.path) }
-                .findAll { !(it.fileName.toFile().name ==~ /(slf4j|logback\-classic)-.*\.jar/) }
-                .findAll { !filesAlreadyInPath.collect { it.getFileName().toString() }.contains(it.fileName.toFile().name)}
-                .each { Files.copy(it, targetPluginPath.resolve(it.fileName), StandardCopyOption.REPLACE_EXISTING) }
-
-        // for the "lib" path include all dependencies
-        getAdditionalDependencies(targetLibPath, artifact).collect { fs.getPath(it.path) }
-                .each { Files.copy(it, targetLibPath.resolve(it.fileName), StandardCopyOption.REPLACE_EXISTING) }
-
-        // the ordering of jars seems to matter in some cases (e.g. neo4j).  the plugin system allows the plugin
-        // to place a Gremlin-Plugin-Paths entry in the jar manifest file to define where specific jar files should
-        // go in the path which provides enough flexibility to control when jars should load.  unfortunately,
-        // this "ordering" issue doesn't seem to be documented as an issue anywhere and it is difficult to say
-        // whether it is a java issue, groovy classloader issue, grape issue, etc.  see this issue for more
-        // on the weirdness: https://github.org/apache/tinkerpop/tinkerpop3/issues/230
-        //
-        // another unfortunate side-effect to this approach is that manual cleanup of jars is kinda messy now
-        // because you can't just delete the plugin directory as one or more of the jars might have been moved.
-        // unsure of what the long term effects of this is.  at the end of the day, users may simply need to
-        // know something about their dependencies in order to have lots of "installed" plugins/dependencies.
-        alterPaths("Gremlin-Plugin-Paths", targetPluginPath, artifact)
-        alterPaths("Gremlin-Lib-Paths", targetLibPath, artifact)
-
-        return "Loaded: " + arguments + (pluginsThatNeedRestart.size() == 0 ? "" : " - restart the console to use $pluginsThatNeedRestart")
-    }
-
-    private static String getPathFromDependency(final Map<String, Object> dep) {
-        return System.getProperty("user.dir") + fileSep + "ext" + fileSep + (String) dep.module
-    }
-
-    private static alterPaths(final String manifestEntry, final Path extPath, final Artifact artifact) {
-        try {
-            // another assumption about the pathing - seems safe for right now as the :install command is
-            // responsible for all this stuff.  if the user chooses to manually install their dependencies
-            // to the console, then it's up to them to sort this stuff out.
-            def pathToInstalled = extPath.resolve(artifact.artifact + "-" + artifact.version + ".jar")
-            final JarFile jar = new JarFile(pathToInstalled.toFile());
-            final Manifest manifest = jar.getManifest()
-
-            // containsKey doesn't seem to want to work - so just check for null - dah
-            def attrLine = manifest.mainAttributes.getValue(manifestEntry)
-            if (attrLine != null) {
-                def splitLine = attrLine.split(";")
-                splitLine.each {
-                    if (it.endsWith("="))
-                        Files.delete(extPath.resolve(it.substring(0, it.length() - 1)))
-                    else {
-                        def kv = it.split("=")
-                        Files.move(extPath.resolve(kv[0]), extPath.resolve(kv[1]), StandardCopyOption.REPLACE_EXISTING)
-                    }
-                }
-            }
-        } catch (Exception ex) {
-            // errors here will likely have to do with bad pathing or poorly constructed entries in the manifest.
-            // hopefully these will only occur for developers of plugins who need to make use of this function.
-            // internally to tinkerpop this is a neo4j-only issue
-            throw new RuntimeException(ex)
-        }
-    }
-
-    private Set<URI> getAdditionalDependencies(final Path extPath, final Artifact artifact) {
         try {
-            def pathToInstalled = extPath.resolve(artifact.artifact + "-" + artifact.version + ".jar")
-            final JarFile jar = new JarFile(pathToInstalled.toFile())
-            final Manifest manifest = jar.getManifest()
-            def attrLine = manifest.mainAttributes.getValue("Gremlin-Plugin-Dependencies")
-            def additionalDependencies = [] as Set<URI>
-            if (attrLine != null) {
-                def splitLine = attrLine.split(";")
-                splitLine.each {
-                    def artifactBits = it.split(":")
-                    def additional = new Artifact(artifactBits[0], artifactBits[1], artifactBits[2])
-
-                    final def additionalDep = makeDepsMap(additional)
-                    additionalDependencies.addAll(Grape.resolve([classLoader: shell.getInterp().getClassLoader()], null, additionalDep))
-                }
-            }
+            def grabber = new DependencyGrabber(shell.getInterp().getClassLoader(), System.getProperty("user.dir") + fileSep + "ext")
+            grabber.copyDependenciesToPath(artifact)
 
-            return additionalDependencies
+            final def dep = grabber.makeDepsMap(artifact)
+            final def pluginsThatNeedRestart = grabDeps(dep)
+            return "Loaded: " + arguments + (pluginsThatNeedRestart.size() == 0 ? "" : " - restart the console to use $pluginsThatNeedRestart")
         } catch (Exception ex) {
-            throw new RuntimeException(ex)
+            return ex.message
         }
     }
 
@@ -220,25 +95,4 @@ class InstallCommand extends CommandSupport {
 
         return new Artifact(group, module, version)
     }
-
-    private def makeDepsMap(final Artifact artifact) {
-        final Map<String, Object> map = new HashMap<>()
-        map.put("classLoader", shell.getInterp().getClassLoader())
-        map.put("group", artifact.getGroup())
-        map.put("module", artifact.getArtifact())
-        map.put("version", artifact.getVersion())
-        map.put("changing", false)
-        return map
-    }
-
-    private static void getFileNames(final List fileNames, final Path dir) {
-        final DirectoryStream<Path> stream = Files.newDirectoryStream(dir)
-        for (Path path : stream) {
-            if (path.toFile().isDirectory()) getFileNames(fileNames, path)
-            else {
-                fileNames.add(path.toAbsolutePath())
-            }
-        }
-        stream.close()
-    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/462fe0a2/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/util/DependencyGrabber.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/util/DependencyGrabber.groovy b/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/util/DependencyGrabber.groovy
index 2e8877d..01b5074 100644
--- a/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/util/DependencyGrabber.groovy
+++ b/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/util/DependencyGrabber.groovy
@@ -21,6 +21,8 @@ package org.apache.tinkerpop.gremlin.groovy.util
 import groovy.grape.Grape
 import org.apache.commons.lang3.SystemUtils
 import org.apache.tinkerpop.gremlin.groovy.plugin.Artifact
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 
 import java.nio.file.*
 import java.util.jar.JarFile
@@ -33,6 +35,7 @@ import java.util.jar.Manifest
  * @author Stephen Mallette (http://stephen.genoprime.com)
  */
 class DependencyGrabber {
+    private static final Logger logger = LoggerFactory.getLogger(DependencyGrabber.class);
 
     private final static String fileSep = System.getProperty("file.separator")
     private final ClassLoader classLoaderToUse
@@ -43,7 +46,7 @@ class DependencyGrabber {
         this.extensionDirectory = extensionDirectory
     }
 
-    def void copyDependenciesToPath(final Artifact artifact) {
+    def Set<String> copyDependenciesToPath(final Artifact artifact) {
         final def dep = makeDepsMap(artifact)
         final String extClassPath = getPathFromDependency(dep)
         final String extLibPath = extClassPath + fileSep + "lib"
@@ -61,22 +64,33 @@ class DependencyGrabber {
         def targetPluginPath = fs.getPath(extPluginPath)
         def targetLibPath = fs.getPath(extLibPath)
 
+        // collect the files already on the path in /lib. making some unfortunate assumptions about what the path
+        // looks like for the gremlin distribution
         def filesAlreadyInPath = []
         def libClassPath
         try {
             libClassPath = fs.getPath(System.getProperty("user.dir") + fileSep + "lib")
             getFileNames(filesAlreadyInPath, libClassPath)
         } catch (Exception ignored) {
-            println "Detected a non-standard Gremlin directory structure during install.  Expecting a 'lib' " +
+            // the user might have a non-standard directory system.  if they are non-standard then they must be
+            // smart and they are therefore capable of resolving their own dependency problems.  this could also
+            // mean that they are running gremlin from source and not from target/*standalone*
+            logger.warn("Detected a non-standard Gremlin directory structure during install.  Expecting a 'lib' " +
                     "directory sibling to 'ext'. This message does not necessarily imply failure, however " +
                     "the console requires a certain directory structure for proper execution. Altering that " +
-                    "structure can lead to unexpected behavior."
+                    "structure can lead to unexpected behavior.")
         }
 
         final def dependencyLocations = [] as Set<URI>
         dependencyLocations.addAll(Grape.resolve([classLoader: this.classLoaderToUse], null, dep))
 
-        // get dependencies for the plugin path which should be part of the class path
+        // for the "plugin" path ignore slf4j related jars.  they are already in the path and will create duplicate
+        // bindings which generate annoying log messages that make you think stuff is wrong.  also, don't bring
+        // over files that are already on the path. these dependencies will be part of the classpath
+        //
+        // additional dependencies are outside those pulled by grape and are defined in the manifest of the plugin jar.
+        // if a plugin uses that setting, it should force "restart" when the plugin is activated.  right now,
+        // it is up to the plugin developer to enforce that setting.
         dependencyLocations.collect(convertUriToPath(fs))
                 .findAll { !(it.fileName.toFile().name ==~ /(slf4j|logback\-classic)-.*\.jar/) }
                 .findAll {!filesAlreadyInPath.collect { it.getFileName().toString() }.contains(it.fileName.toFile().name)}
@@ -90,6 +104,17 @@ class DependencyGrabber {
         dependencyLocations.collect(convertUriToPath(fs)).each(copyTo(targetLibPath))
         getAdditionalDependencies(targetLibPath, artifact).collect(convertUriToPath(fs)).each(copyTo(targetLibPath))
 
+        // the ordering of jars seems to matter in some cases (e.g. neo4j).  the plugin system allows the plugin
+        // to place a Gremlin-Plugin-Paths entry in the jar manifest file to define where specific jar files should
+        // go in the path which provides enough flexibility to control when jars should load.  unfortunately,
+        // this "ordering" issue doesn't seem to be documented as an issue anywhere and it is difficult to say
+        // whether it is a java issue, groovy classloader issue, grape issue, etc.  see this issue for more
+        // on the weirdness: https://github.org/apache/tinkerpop/tinkerpop3/issues/230
+        //
+        // another unfortunate side-effect to this approach is that manual cleanup of jars is kinda messy now
+        // because you can't just delete the plugin directory as one or more of the jars might have been moved.
+        // unsure of what the long term effects of this is.  at the end of the day, users may simply need to
+        // know something about their dependencies in order to have lots of "installed" plugins/dependencies.
         alterPaths("Gremlin-Plugin-Paths", targetPluginPath, artifact)
         alterPaths("Gremlin-Lib-Paths", targetLibPath, artifact)
     }
@@ -100,7 +125,7 @@ class DependencyGrabber {
             def copying = path.resolve(p.fileName)
             if (!copying.toFile().exists()) {
                 Files.copy(p, copying, StandardCopyOption.REPLACE_EXISTING)
-                println "Copying - $copying"
+                logger.info("Copying - $copying")
             }
         }
     }
@@ -163,11 +188,10 @@ class DependencyGrabber {
     }
 
     private String getPathFromDependency(final Map<String, Object> dep) {
-        def fileSep = System.getProperty("file.separator")
         return this.extensionDirectory + fileSep + (String) dep.module
     }
 
-    private def makeDepsMap(final Artifact artifact) {
+    public def makeDepsMap(final Artifact artifact) {
         final Map<String, Object> map = new HashMap<>()
         map.put("classLoader", this.classLoaderToUse)
         map.put("group", artifact.getGroup())
@@ -177,7 +201,7 @@ class DependencyGrabber {
         return map
     }
 
-    private static void getFileNames(final List fileNames, final Path dir) {
+    public static void getFileNames(final List fileNames, final Path dir) {
         final DirectoryStream<Path> stream = Files.newDirectoryStream(dir)
         for (Path path : stream) {
             if (path.toFile().isDirectory()) getFileNames(fileNames, path)