You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2019/06/06 10:41:08 UTC
[lucene-solr] branch jira/SOLR-13452_gradle_3 updated (b74abe2 ->
c4ccba8)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a change to branch jira/SOLR-13452_gradle_3
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from b74abe2 SOLR-13452: Some cleanup, add a new task to help find unused deps, clean up some deps.
new 5d258df SOLR-13452: Tweak main build.gradle a bit.
new c4ccba8 SOLR-13452: Improve unused dep checker to not count on creating the dist tgz and zip first and check if jars are used by other dep jars even if not by the module itself.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build.gradle | 30 +--
.../org/apache/lucene/gradle/UnusedDeps.groovy | 274 +++++++++++++++------
lucene/codecs/build.gradle | 2 -
solr/build.gradle | 1 -
4 files changed, 212 insertions(+), 95 deletions(-)
[lucene-solr] 02/02: SOLR-13452: Improve unused dep checker to not
count on creating the dist tgz and zip first and check if jars are used by
other dep jars even if not by the module itself.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch jira/SOLR-13452_gradle_3
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit c4ccba87ba33d50a00e039ad96afeb576b30d20c
Author: markrmiller <ma...@apache.org>
AuthorDate: Thu Jun 6 05:40:52 2019 -0500
SOLR-13452: Improve unused dep checker to not count on creating the dist tgz and zip first and check if jars are used by other dep jars even if not by the module itself.
---
.../org/apache/lucene/gradle/UnusedDeps.groovy | 274 +++++++++++++++------
lucene/codecs/build.gradle | 2 -
solr/build.gradle | 1 -
3 files changed, 196 insertions(+), 81 deletions(-)
diff --git a/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy b/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy
index 9a9c123..aed0703 100644
--- a/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy
+++ b/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy
@@ -23,6 +23,7 @@ import org.gradle.api.DefaultTask
import org.gradle.api.Project
import org.gradle.api.artifacts.Configuration
import org.gradle.api.file.RelativePath
+import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.InputDirectory
@@ -36,102 +37,219 @@ import java.util.regex.Pattern
// dev util task to help find possible unused deps
class UnusedDeps extends DefaultTask {
- protected static Pattern pattern = Pattern.compile("\\((.*?\\.jar)\\)")
+ protected static Pattern pattern = Pattern.compile("\\(([^\\s]*?\\.jar)\\)")
+
+ protected configuration = "runtimeClasspath"
+ protected File tmpDir
+ protected File distDir
+ protected File jdepsDir
- @Inject
public UnusedDeps() {
-
+ //!project.getPlugins().hasPlugin('java-base') ||
+ if (!project.configurations.hasProperty('runtimeClasspath')) {
+ return
+ }
+
+ if (project.hasProperty('unusedDepsConfig')) {
+ configuration = project.unusedDepsConfig
+ }
+
+ Configuration config = project.configurations[this.configuration]
+
+ List<Project> buildProjects = new ArrayList()
+ buildProjects.add(project)
+ config.getAllDependencies().forEach({ dep ->
+ if (dep instanceof DefaultProjectDependency) {
+ Project dProject = dep.getDependencyProject()
+ buildProjects.add(dProject)
+ }
+ })
+
+ project.tasks.create(name: "export", type: org.gradle.api.tasks.Copy) {
+ outputs.upToDateWhen { false }
+ into({ makeDirs(); distDir })
+ buildProjects.each {subproject ->
+ project.evaluationDependsOn(subproject.path)
+ def topLvlProject
+ if (subproject.group ==~ /.*?\.lucene(?:\.\w+)?/) {
+ topLvlProject = project.project(":lucene")
+ } else if (subproject.group ==~ /.*?\.solr(?:\.\w+)?/) {
+ topLvlProject = project.project(":solr")
+ }
+
+ if (subproject.getPlugins().hasPlugin(PartOfDist) && subproject.tasks.findByName('jar') && subproject.configurations.hasProperty('runtimeClasspath')) {
+ from(subproject.jar.outputs.files) {
+ include "*.jar"
+ into ({topLvlProject.name + '/' + topLvlProject.relativePath(subproject.projectDir)})
+ }
+ def files = { getFiles(subproject) }
+ from(files) {
+ include "*.jar"
+ into ({topLvlProject.name + '/' + topLvlProject.relativePath(subproject.projectDir) + "/lib"})
+ }
+ }
+ }
+
+ includeEmptyDirs = false
+ }
+
+ dependsOn project.tasks.export
+ }
+
+ protected void makeDirs() {
+ tmpDir = File.createTempDir()
+ tmpDir.deleteOnExit()
+ tmpDir.mkdirs()
+ distDir = new File(tmpDir, 'distDir')
+ jdepsDir = new File(tmpDir, 'jdepsDir')
+ distDir.mkdirs()
+ jdepsDir.mkdirs()
+ }
+
+ private static Collection getFiles(Project subproject) {
+ def files = subproject.configurations.runtimeClasspath.files
+ if (!subproject.name.equals('solr-core') && subproject.path.startsWith(":solr:contrib:")) {
+ subproject.evaluationDependsOn(subproject.rootProject.project(":solr:solr-core").path)
+ files = files - subproject.rootProject.project(":solr:solr-core").configurations.runtimeClasspath.files
+ files = files - subproject.rootProject.project(":solr:solr-core").jar.outputs.files
+ }
+ if (!subproject.name.equals('lucene-core') && subproject.path.startsWith(":lucene:")) {
+ subproject.evaluationDependsOn(subproject.rootProject.project(":lucene:lucene-core").path)
+ files = files - subproject.rootProject.project(":lucene:lucene-core").configurations.runtimeClasspath.files
+ files = files - subproject.rootProject.project(":lucene:lucene-core").jar.outputs.files
+ }
+ return files
}
@TaskAction
void execute() {
// make sure ant task logging shows up by default
ant.lifecycleLogLevel = "INFO"
- String configuration = "runtimeClasspath"
- if (project.hasProperty('unusedDepsConfig')) {
- configuration = project.unusedDepsConfig
+ Project topLvlProject
+
+ if (project.group ==~ /.*?\.lucene(?:\.\w+)?/) {
+ topLvlProject = project.project(":lucene")
+ } else if (project.group ==~ /.*?\.solr(?:\.\w+)?/) {
+ topLvlProject = project.project(":solr")
}
- File tmpDir = File.createTempDir()
- tmpDir.deleteOnExit()
- tmpDir.mkdirs()
+ def luceneDist = new File("/data1/mark/tmp", "lucene")
+ def solrDist = new File("/data1/mark/tmp", "solr")
- File distDir = new File(tmpDir, 'distDir')
- distDir.mkdirs()
- File jdepsDir = new File(tmpDir, 'jdepsDir')
- jdepsDir.mkdirs()
+ Configuration config = project.configurations[this.configuration]
-
- Project topLvlProject
- File destinationDir
-
- if (project.group ==~ /.*?\.lucene(?:\.\w+)?/) {
- topLvlProject = project.project(":lucene")
- destinationDir = new File(topLvlProject.projectDir, "dist")
- } else if (project.group ==~ /.*?\.solr(?:\.\w+)?/) {
- topLvlProject = project.project(":solr")
- destinationDir = new File(topLvlProject.projectDir, "dist")
- }
-
-
- ant.untar(src: "${destinationDir}/${topLvlProject.name}-${topLvlProject.version}.tgz", dest: distDir.getAbsolutePath(), compression:"gzip")
-
- def distPath = "${distDir}/" + topLvlProject.relativePath(project.projectDir)
-
- ant.exec (executable: "jdeps", failonerror: true, resolveexecutable: true) {
- ant.arg(line: '--class-path ' + "${distPath}/lib/" + '*')
- ant.arg(line: '--multi-release 11')
- ant.arg(value: '-recursive')
- ant.arg(value: '-verbose:class')
- ant.arg(line: "-dotoutput ${jdepsDir.getAbsolutePath()}")
- ant.arg(value: "${distPath}/${project.name}-${topLvlProject.version}.jar")
- }
-
-
- Set<String> usedDepJarNames = new HashSet<>()
- File file = new File(jdepsDir, "${project.name}-${topLvlProject.version}.jar.dot")
- def lines = file.readLines()
-
- lines.each { String line ->
- Matcher m = pattern.matcher(line)
- if (m.find()) {
- usedDepJarNames.add(m.group(1))
+ Set<String> usedDepJarNames = getDirectlyUsedDeps(topLvlProject, project, distDir, jdepsDir)
+
+ Set<File> ourDeps = getDeps(project, config)
+
+ config.getAllDependencies().forEach({ dep ->
+ if (dep instanceof DefaultProjectDependency) {
+ Project dProject = dep.getDependencyProject()
+ def depTopLvlProject
+ if (dProject.group ==~ /.*?\.lucene(?:\.\w+)?/) {
+ depTopLvlProject = project.project(":lucene")
+ } else if (dProject.group ==~ /.*?\.solr(?:\.\w+)?/) {
+ depTopLvlProject = project.project(":solr")
}
+
+ Set<String> projectUsedDeps = getDirectlyUsedDeps(depTopLvlProject, dProject, distDir, jdepsDir)
+
+ usedDepJarNames += projectUsedDeps
}
-
- Set<File> ourDeps = new HashSet<>()
-
- Configuration config = project.configurations[configuration]
- if (config.isCanBeResolved()) {
- config.getResolvedConfiguration().getResolvedArtifacts().forEach( { ra -> ourDeps.add(ra.getFile()) })
- // exclude our jar and jarTest outputs
- def ourJars = new ArrayList()
- project.rootProject.subprojects.each{ subproject ->
- if (subproject.hasProperty('jar')) {
- ourJars.addAll(subproject.jar.outputs.files)
- if (subproject.hasProperty('jarTest')) {
- ourJars.addAll(subproject.jarTest.outputs.files)
- }
- }
+ })
+
+ usedDepJarNames -= ["${project.name}-${project.version}.jar"]
+
+ Set<String> ourDepJarNames = new HashSet<>()
+ ourDeps.forEach( { ourDepJarNames.add(it.getName()) } )
+
+ Set<String> unusedJarNames = new HashSet<>()
+ unusedJarNames.addAll(ourDepJarNames)
+ unusedJarNames -= usedDepJarNames
+ unusedJarNames = unusedJarNames.toSorted()
+
+ Set<String> foundDeps = new HashSet<>()
+
+ File jdepsLucene = new File(jdepsDir, "lucene")
+ File jdepsSolr = new File(jdepsDir, "solr")
+
+ for (File file : jdepsLucene.listFiles()) {
+ lookForDep(file, foundDeps)
+ }
+ for (File file : jdepsSolr.listFiles()) {
+ lookForDep(file, foundDeps)
+ }
+
+ println ''
+ println 'Our classpath dependency count ' + ourDepJarNames.size()
+ println 'Our directly used dependency count ' + usedDepJarNames.size()
+ println ''
+ println 'List of possibly unused jars - they may be used at runtime however (Class.forName or something), this is not definitive.'
+ println 'We take our classpath dependenies, substract our direct dependencies and then subtract dependencies used by our direct dependencies'
+ println ''
+
+ unusedJarNames.forEach({
+ if (!foundDeps.contains(it)) {
+ println it
+ }
+ })
+
+ project.delete(tmpDir)
+ }
+
+ protected Set getDeps(Project project, Configuration config) {
+ Set<File> ourDeps = new HashSet<>()
+
+ if (config.isCanBeResolved()) {
+ config.getResolvedConfiguration().getResolvedArtifacts().forEach( { ra -> ourDeps.add(ra.getFile()) })
+ }
+ return ourDeps
+ }
+
+ protected Set getDirectlyUsedDeps(Project topLvlProject, Project project, File distDir, File jdepsDir) {
+ def distPath = "${distDir}/" + topLvlProject.name + "/" + topLvlProject.relativePath(project.projectDir)
+ def dotOutPath = jdepsDir.getAbsolutePath() + "/" + topLvlProject.name + "/" + "${project.name}-${project.version}"
+
+ ant.exec (executable: "jdeps", failonerror: true, resolveexecutable: true) {
+ ant.arg(line: '--class-path ' + "${distPath}/lib/" + '*')
+ ant.arg(line: '--multi-release 11')
+ ant.arg(value: '-recursive')
+ ant.arg(value: '-verbose:class')
+ ant.arg(line: "-dotoutput ${dotOutPath}")
+ ant.arg(value: "${distPath}/${project.name}-${project.version}.jar")
+ }
+
+ File dotFile = new File(jdepsDir, topLvlProject.name + "/" + "${project.name}-${project.version}/${project.name}-${project.version}.jar.dot")
+ Set<String> usedDepJarNames = getUsedJars(project, dotFile)
+
+ return usedDepJarNames
+ }
+
+ protected Set getUsedJars(Project project, File dotFile) {
+ Set<String> usedDepJarNames = new HashSet<>()
+
+ def lines = dotFile.readLines()
+ String lastName = ""
+ for (String line : lines) {
+ Matcher m = pattern.matcher(line)
+ if (m.find()) {
+ String jarName = m.group(1)
+ if (!lastName.equals(jarName)) {
+ usedDepJarNames.add(jarName)
}
- ourDeps = ourDeps - ourJars
+ lastName = jarName
}
+ }
+ return usedDepJarNames
+ }
+
+ protected void lookForDep(File dir, Set<String> foundDeps) {
+ dir.eachFile() {
+ Set<String> usedDepJarNames = getUsedJars(project, it)
+ foundDeps.addAll(usedDepJarNames)
- Set<String> ourDepJarNames = new HashSet<>()
- ourDeps.forEach( { ourDepJarNames.add(it.getName()) } )
-
- println 'our dep count ' + ourDepJarNames.size()
- println 'used count ' + usedDepJarNames.size()
-
- Set<String> unusedJarNames = new HashSet<>()
- unusedJarNames.addAll(ourDepJarNames)
- unusedJarNames -= usedDepJarNames
- unusedJarNames = unusedJarNames.toSorted()
- unusedJarNames.forEach( { println it } )
-
- project.delete(tmpDir)
-
+ }
}
}
diff --git a/lucene/codecs/build.gradle b/lucene/codecs/build.gradle
index 5c7afee..a29c64f 100644
--- a/lucene/codecs/build.gradle
+++ b/lucene/codecs/build.gradle
@@ -19,8 +19,6 @@ apply plugin: 'java-library'
apply plugin: 'maven-publish'
apply plugin: org.apache.lucene.gradle.PartOfDist
-archivesBaseName = 'codecs'
-
dependencies {
implementation project(':lucene:lucene-core')
diff --git a/solr/build.gradle b/solr/build.gradle
index 87f957c..9325c49 100644
--- a/solr/build.gradle
+++ b/solr/build.gradle
@@ -36,5 +36,4 @@ task packageDist(type: org.apache.lucene.gradle.PackageLuceneSolrDist) {
subprojects {
tasks.create("unusedDeps", org.apache.lucene.gradle.UnusedDeps)
- unusedDeps.dependsOn packageDist
}
[lucene-solr] 01/02: SOLR-13452: Tweak main build.gradle a bit.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch jira/SOLR-13452_gradle_3
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 5d258df02bbc3ae6071f659d66bb435c98510f6a
Author: markrmiller <ma...@apache.org>
AuthorDate: Thu Jun 6 01:00:10 2019 -0500
SOLR-13452: Tweak main build.gradle a bit.
---
build.gradle | 30 ++++++++++++++++--------------
1 file changed, 16 insertions(+), 14 deletions(-)
diff --git a/build.gradle b/build.gradle
index 496aa6b..147d3fe 100644
--- a/build.gradle
+++ b/build.gradle
@@ -37,6 +37,7 @@ ant.lifecycleLogLevel = "INFO"
// -> lucene-solr all project config
// -> lucene-solr sub module config
// -> lucene-solr root project config
+// -> all projects config
// -> lucene-solr IDE config
// -> other config
@@ -153,9 +154,11 @@ configure(rootProject) {
}
-// -> lucene-solr IDE config - setup eclipse and idea
-
+// -> all projects config
configure(allprojects) {
+
+ // -> lucene-solr IDE config - setup eclipse and idea
+
apply plugin: 'eclipse'
apply plugin: 'idea'
@@ -163,18 +166,14 @@ configure(allprojects) {
project.apply from: new File(rootProjectDir, "buildSrc/ide/eclipse.gradle")
project.apply from: new File(rootProjectDir, "buildSrc/ide/idea.gradle")
}
-}
-
-// -> other config
-
-configure(allprojects) {
-
- dependencies {
- configurations.all {
- // stax-api classes are in Java 11
- exclude group: 'javax.xml.stream', module: 'stax-api'
- exclude group: 'stax', module: 'stax-api'
- exclude group: 'stax', module: 'stax'
+
+ // dependencies block that applies to all projects
+ dependencies {
+ configurations.all {
+ // stax-api classes are in Java 11
+ exclude group: 'javax.xml.stream', module: 'stax-api'
+ exclude group: 'stax', module: 'stax-api'
+ exclude group: 'stax', module: 'stax'
}
modules {
@@ -183,6 +182,9 @@ configure(allprojects) {
}
}
+// -> other config
+
+
// Single JavaDocs for all modules
def noJavaDocModules = ["buildSrc", "dev-tools","lucene", "solr", "solr-ref-guide", "lucene-analysis", "lucene-backward-codecs", "solr-contrib", "solr-example", "solr-example-DIH"]