You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2019/06/10 05:58:16 UTC
[lucene-solr] branch jira/SOLR-13452_gradle_3 updated: SOLR-13452:
Some more work towards a solid transitive dependency solution.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch jira/SOLR-13452_gradle_3
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/SOLR-13452_gradle_3 by this push:
new 4780c1a SOLR-13452: Some more work towards a solid transitive dependency solution.
4780c1a is described below
commit 4780c1ad56c9ce856f96d3ae7d2a54fac71ba68c
Author: markrmiller <ma...@apache.org>
AuthorDate: Mon Jun 10 00:57:59 2019 -0500
SOLR-13452: Some more work towards a solid transitive dependency solution.
---
build.gradle | 24 ++++++-----
.../org/apache/lucene/gradle/FindInSrc.groovy | 12 ++----
.../gradle/LuceneSolrForbiddenApisPlugin.groovy | 1 +
.../org/apache/lucene/gradle/MissingDeps.groovy | 7 ----
.../org/apache/lucene/gradle/UnusedDeps.groovy | 47 +++++++++++-----------
solr/contrib/dataimporthandler/build.gradle | 3 ++
solr/core/build.gradle | 29 ++++++++-----
versions.lock | 3 +-
8 files changed, 67 insertions(+), 59 deletions(-)
diff --git a/build.gradle b/build.gradle
index 17e5fdc..9414fa4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -198,24 +198,30 @@ configure(allprojects) {
description = "Lists dependencies that may be unused for a module."
project.tasks.create("jdepsReport", org.apache.lucene.gradle.JdepsReport, jdepsReportDir, true)
- project.tasks.create("iunusedDeps", org.apache.lucene.gradle.UnusedDeps, jdepsReportDir)
- project.iunusedDeps.dependsOn project.jdepsReport
- project.iunusedDeps.doLast {
- //delete(jdepsReportDir)
+ task unusedDeps(type: org.apache.lucene.gradle.UnusedDeps) {
+ inputDirectory jdepsReportDir
}
- dependsOn iunusedDeps
+ project.unusedDeps.dependsOn project.jdepsReport
+ project.unusedDeps.doLast {
+ delete(jdepsReportDir)
+ }
+ dependsOn unusedDeps
}
+
task missingDependencies {
group = 'Help'
description = "Lists classes from this module with missing runtime dependencies (we ignore scanning some root deps (ie hadoop) and some violations (ie annotations)."
project.tasks.create("jdepsReportR", org.apache.lucene.gradle.JdepsReport, jdepsReportDir, true)
// we exclude hadoop and annotations
- project.tasks.create("imissingDeps", org.apache.lucene.gradle.MissingDeps, jdepsReportDir, ['**/*hadoop*.dot'], ['org\\.apache\\.yetus\\.audience\\.InterfaceAudience', 'javax\\.annotation\\..*', 'org\\.apache\\.yetus\\.audience', 'com\\.google\\.errorprone\\.annotations\\..*'])
- project.imissingDeps.dependsOn project.jdepsReportR
- project.imissingDeps.doLast {
+ task missingDeps(type: org.apache.lucene.gradle.MissingDeps) {
+ inputDirectory jdepsReportDir
+ }
+
+ project.missingDeps.dependsOn project.jdepsReportR
+ project.missingDeps.doLast {
delete(jdepsReportDir)
}
- dependsOn imissingDeps
+ dependsOn missingDeps
}
}
diff --git a/buildSrc/src/main/groovy/org/apache/lucene/gradle/FindInSrc.groovy b/buildSrc/src/main/groovy/org/apache/lucene/gradle/FindInSrc.groovy
index 807193d..0137d89 100644
--- a/buildSrc/src/main/groovy/org/apache/lucene/gradle/FindInSrc.groovy
+++ b/buildSrc/src/main/groovy/org/apache/lucene/gradle/FindInSrc.groovy
@@ -46,9 +46,7 @@ class FindInSrc {
protected static Pattern pattern = Pattern.compile("->\\s\\\"([^\\s]*?)\\s")
protected static Pattern srcJar = Pattern.compile("(.*?)-sources.jar")
protected static Pattern dotFilePattern = Pattern.compile("(.*?).jar.dot")
-
-
-
+
public FindInSrc() {
}
@@ -71,7 +69,8 @@ class FindInSrc {
Stream.of(listSrcFiles.toArray())
.parallel()
.forEach( { file ->
- if (!file.name.endsWith('-sources.jar')) return
+ // if (!file.name.endsWith('-sources.jar')) return
+ // if (foundInsrc.get()) return // stop after we find first occurence(s)
Matcher nameMatcher = srcJar.matcher(file.name)
if (nameMatcher.matches()) {
String artifactName = nameMatcher.group(1)
@@ -123,8 +122,3 @@ class FindInSrc {
}
}
-
-
-
-
-
diff --git a/buildSrc/src/main/groovy/org/apache/lucene/gradle/LuceneSolrForbiddenApisPlugin.groovy b/buildSrc/src/main/groovy/org/apache/lucene/gradle/LuceneSolrForbiddenApisPlugin.groovy
index 1a9bab3..4aae479 100644
--- a/buildSrc/src/main/groovy/org/apache/lucene/gradle/LuceneSolrForbiddenApisPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/lucene/gradle/LuceneSolrForbiddenApisPlugin.groovy
@@ -32,6 +32,7 @@ class LuceneSolrForbiddenApisPlugin implements Plugin<Project> {
}
project.tasks.withType(CheckForbiddenApis) { task ->
+ task.failOnUnresolvableSignatures = false
task.bundledSignatures = ['jdk-unsafe', 'jdk-deprecated', 'jdk-non-portable', 'jdk-reflection']
task.signaturesURLs = [ getClass().getResource('/forbidden/base.txt') ]
diff --git a/buildSrc/src/main/groovy/org/apache/lucene/gradle/MissingDeps.groovy b/buildSrc/src/main/groovy/org/apache/lucene/gradle/MissingDeps.groovy
index 824773b..36039fc 100644
--- a/buildSrc/src/main/groovy/org/apache/lucene/gradle/MissingDeps.groovy
+++ b/buildSrc/src/main/groovy/org/apache/lucene/gradle/MissingDeps.groovy
@@ -61,13 +61,6 @@ class MissingDeps extends DefaultTask {
protected configuration = "runtimeClasspath"
- @Inject
- public MissingDeps(File inputDirectory, List<String> depExcludes, List<String> classExcludes) {
- this.inputDirectory = inputDirectory
- this.depExcludes = depExcludes
- this.classExcludes = classExcludes
- }
-
@TaskAction
void execute() {
diff --git a/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy b/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy
index 441326a..45c1aba 100644
--- a/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy
+++ b/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy
@@ -20,6 +20,7 @@ import org.gradle.api.artifacts.ResolvedArtifact
import org.gradle.api.artifacts.ResolvedDependency
import javax.inject.Inject
import org.gradle.api.DefaultTask
+import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.artifacts.Configuration
import org.gradle.api.artifacts.Dependency
@@ -48,18 +49,15 @@ class UnusedDeps extends DefaultTask {
protected static Pattern pattern = Pattern.compile("\\(([^\\s]*?\\.jar)\\)")
protected configuration = "runtimeClasspath"
- protected File distDir
- protected File jdepsDir
@InputDirectory
File inputDirectory
- @Inject
- public UnusedDeps(File inputDirectory) {
- this.inputDirectory = inputDirectory
-
- distDir = new File(inputDirectory, 'distDir')
- jdepsDir = new File(inputDirectory, 'jdepsDir')
+ @Input
+ @Optional
+ List<String> jarExcludes
+
+ public UnusedDeps() {
if (project.hasProperty('useConfiguration')) {
configuration = project.useConfiguration
@@ -68,6 +66,13 @@ class UnusedDeps extends DefaultTask {
if (!project.configurations.hasProperty(configuration)) {
return
}
+ }
+
+ @TaskAction
+ void execute() {
+
+ // make sure ant task logging shows up by default
+ ant.lifecycleLogLevel = "INFO"
Configuration config = project.configurations[this.configuration]
@@ -79,18 +84,12 @@ class UnusedDeps extends DefaultTask {
buildProjects.add(dProject)
}
})
- }
-
- @TaskAction
- void execute() {
- // make sure ant task logging shows up by default
- ant.lifecycleLogLevel = "INFO"
+ File distDir = new File(inputDirectory, 'distDir')
+ File jdepsDir = new File(inputDirectory, 'jdepsDir')
def topLvlProject = getTopLvlProject(project)
- Configuration config = project.configurations[this.configuration]
-
Set<String> usedStaticallyJarNames = getStaticallyReferencedDeps(topLvlProject, project, distDir, jdepsDir)
Set<File> ourDeps = getAllDefinedDeps(project, config)
@@ -142,6 +141,8 @@ class UnusedDeps extends DefaultTask {
println 'Direct deps that may be unused:'
+ boolean failTask = false;
+
unusedJarNames.forEach({
if (!depsInDirectUse.contains(it) && ourImmediatelyDefinedDeps.contains(it)) {
@@ -149,6 +150,7 @@ class UnusedDeps extends DefaultTask {
if (findInSrc(it)) {
println ' - ' + it + ' *'
} else {
+ failTask = true
println ' - ' + it
}
@@ -159,13 +161,13 @@ class UnusedDeps extends DefaultTask {
println 'Deps brought in by other modules that may be unused in this module:'
unusedJarNames.forEach({
if (!depsInDirectUse.contains(it) && !ourImmediatelyDefinedDeps.contains(it)) {
- if (findInSrc(it)) {
- println ' - ' + it + ' *'
- } else {
- println ' - ' + it
- }
+ println ' - ' + it
}
})
+
+ if (failTask) {
+ throw new GradleException("Unused dependencies found! Remove them or add an exclusion if they are actually necessary.")
+ }
}
static class NonProjectSpec implements Spec<Dependency> {
@@ -228,8 +230,7 @@ class UnusedDeps extends DefaultTask {
depsInDirectUse.addAll(getDirectlyUsedJars(project, it))
}
}
-
-
+
protected boolean findInSrc(String jarName) {
AtomicBoolean foundInsrc = new AtomicBoolean(false)
def files = project.configurations[configuration].resolvedConfiguration.getFiles()
diff --git a/solr/contrib/dataimporthandler/build.gradle b/solr/contrib/dataimporthandler/build.gradle
index a673846..2ad7155 100644
--- a/solr/contrib/dataimporthandler/build.gradle
+++ b/solr/contrib/dataimporthandler/build.gradle
@@ -42,6 +42,9 @@ dependencies {
implementation ('org.slf4j:slf4j-api') { transitive = false }
implementation ('commons-io:commons-io') { transitive = false }
+ implementation ('com.google.guava:guava') {
+ exclude group: '*', module: '*' // many annotation deps we don't need
+ }
implementation ('org.apache.httpcomponents:httpclient') { transitive = false }
implementation ('org.apache.zookeeper:zookeeper') { transitive = false }
implementation ('io.dropwizard.metrics:metrics-core') { transitive = false }
diff --git a/solr/core/build.gradle b/solr/core/build.gradle
index c41b409..b0adda1 100644
--- a/solr/core/build.gradle
+++ b/solr/core/build.gradle
@@ -44,6 +44,9 @@ dependencies {
implementation project(':lucene:lucene-join')
implementation project(':solr:solr-solrj')
+ // used for log4j2 async support
+ runtimeOnly ('com.lmax:disruptor')
+
// kerb, kerby, and curator deps used for security and auth with zk/hdfs/kerberos
runtimeOnly ('org.apache.curator:curator-recipes') {
exclude group: '*', module: '*' // brings in a lot we don't use
@@ -53,16 +56,15 @@ dependencies {
runtimeOnly ('org.apache.kerby:kerby-asn1')
runtimeOnly ('org.apache.kerby:kerby-pkix')
- // used by calicte.avatica
- runtimeOnly ('com.google.protobuf:protobuf-java')
+ runtimeOnly ('com.google.protobuf:protobuf-java') // used by calicte.avatica
+ runtimeOnly ('org.slf4j:jcl-over-slf4j') // bridge java common logging to slf4j
compileOnly ('javax.servlet:javax.servlet-api')
- // TODO: this is not great
+ // TODO: this is not great, still have to work out logging dist strat
implementation ('org.apache.logging.log4j:log4j-slf4j-impl')
implementation ('org.slf4j:slf4j-api')
- implementation ('org.slf4j:jcl-over-slf4j')
implementation ('org.apache.zookeeper:zookeeper') {
exclude group: '*', module: '*' // zk has many deps we don't need
}
@@ -78,13 +80,13 @@ dependencies {
}
implementation ('io.dropwizard.metrics:metrics-jvm')
implementation ('io.dropwizard.metrics:metrics-jmx')
- api ('commons-codec:commons-codec') // api because of forbidden apis :(
+ implementation ('commons-codec:commons-codec')
implementation ('commons-io:commons-io')
implementation ('org.apache.commons:commons-exec')
implementation ('commons-fileupload:commons-fileupload')
implementation ('commons-cli:commons-cli')
implementation ('org.apache.commons:commons-text')
- api ('com.google.guava:guava') { // api because of forbidden apis :(
+ implementation ('com.google.guava:guava') {
exclude group: '*', module: '*' // many annotation deps we don't need
}
implementation ('org.locationtech.spatial4j:spatial4j')
@@ -92,13 +94,11 @@ dependencies {
implementation ('org.ow2.asm:asm')
implementation ('org.restlet.jee:org.restlet')
- implementation ('org.restlet.jee:org.restlet.ext.servlet')
implementation ('com.carrotsearch:hppc')
implementation ('org.apache.logging.log4j:log4j-api')
implementation ('org.apache.logging.log4j:log4j-core')
implementation ('org.apache.logging.log4j:log4j-1.2-api')
- implementation ('org.slf4j:jcl-over-slf4j')
implementation ('com.fasterxml.jackson.core:jackson-core')
implementation ('com.fasterxml.jackson.core:jackson-databind')
@@ -153,7 +153,7 @@ dependencies {
implementation ('org.eclipse.jetty:jetty-servlet')
implementation ('org.eclipse.jetty:jetty-server')
- testRuntimeOnly ('com.lmax:disruptor')
+ testImplementation ('org.restlet.jee:org.restlet.ext.servlet')
// kerby
testRuntimeOnly ('org.apache.kerby:kerb-core')
@@ -211,6 +211,17 @@ forbiddenApisTest {
exclude 'org/apache/hadoop/**'
}
+missingDeps() {
+ depExcludes = ['**/*hadoop*.dot'] // we exclude examining hadoop jars because we don't need lots of these deps
+
+ classExcludes = [
+ 'org\\.apache\\.yetus\\.audience\\.InterfaceAudience', // annotations jar
+ 'javax\\.annotation\\..*', // annotations jar
+ 'org\\.apache\\.yetus\\.audience', // annotations jar
+ 'com\\.google\\.errorprone\\.annotations\\..*' // annotations jar
+ ]
+}
+
task javacc(type: org.apache.lucene.gradle.SolrQPJavaCC) {
inputFile file('src/java/org/apache/solr/parser/QueryParser.jj')
target file('src/java/org/apache/solr/parser')
diff --git a/versions.lock b/versions.lock
index 85b338b..56f6ca8 100644
--- a/versions.lock
+++ b/versions.lock
@@ -210,8 +210,7 @@ org.ow2.asm:asm-analysis:6.2 (1 constraints: e309d6a5)
org.ow2.asm:asm-commons:6.2 (1 constraints: aa04272c)
org.ow2.asm:asm-tree:6.2 (2 constraints: 2d14228c)
org.quartz-scheduler:quartz:2.2.0 (1 constraints: 2106eb4d)
-org.restlet.jee:org.restlet:2.3.0 (2 constraints: e3159ee6)
-org.restlet.jee:org.restlet.ext.servlet:2.3.0 (1 constraints: 0705fe35)
+org.restlet.jee:org.restlet:2.3.0 (1 constraints: 0705fe35)
org.rrd4j:rrd4j:3.5 (1 constraints: ac04252c)
org.simpleframework:simple-xml:2.7.1 (3 constraints: bd20563c)
org.slf4j:jcl-over-slf4j:1.7.25 (4 constraints: f629cf87)