You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2019/12/06 16:08:25 UTC
[lucene-solr] branch gradle-master updated (de8a37e -> 3e4d8a1)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a change to branch gradle-master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from de8a37e Adding woodstox temporarily so that tests pass.
add dad933b SOLR-14001: fix HdfsBackupRepositoryTest on windows
add 9ec8a86 LUCENE-9077: Add support for running under gradle test runner.
new daa0779 Merge remote-tracking branch 'origin/master' into gradle-master
new 3e4d8a1 Initial support for running with security manager (lucene).
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
gradle/testing/defaults-tests.gradle | 4 ---
.../testing/policies}/tests.policy | 32 +++++++++++------
gradle/testing/randomization.gradle | 42 ++++++++++++++++++++--
.../apache/lucene/util/TestSecurityManager.java | 4 ++-
lucene/tools/junit4/solr-tests.policy | 1 +
5 files changed, 64 insertions(+), 19 deletions(-)
copy {lucene/tools/junit4 => gradle/testing/policies}/tests.policy (86%)
[lucene-solr] 02/02: Initial support for running with security
manager (lucene).
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch gradle-master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 3e4d8a17acd6c0de8dc8e87bdfb708289fc1c437
Author: Dawid Weiss <dw...@apache.org>
AuthorDate: Fri Dec 6 17:08:14 2019 +0100
Initial support for running with security manager (lucene).
---
gradle/testing/defaults-tests.gradle | 4 --
gradle/testing/policies/tests.policy | 116 +++++++++++++++++++++++++++++++++++
gradle/testing/randomization.gradle | 42 ++++++++++++-
3 files changed, 155 insertions(+), 7 deletions(-)
diff --git a/gradle/testing/defaults-tests.gradle b/gradle/testing/defaults-tests.gradle
index 8a6860c..7997916 100644
--- a/gradle/testing/defaults-tests.gradle
+++ b/gradle/testing/defaults-tests.gradle
@@ -16,10 +16,6 @@ allprojects {
maxParallelForks = propertyOrDefault("tests.jvms", (int) Math.max(1, Math.min(Runtime.runtime.availableProcessors() / 2.0, 4.0)))
- if (Boolean.parseBoolean(propertyOrDefault("tests.failfast", 'false'))) {
- failFast true
- }
-
minHeapSize = "256m"
maxHeapSize = "512m"
diff --git a/gradle/testing/policies/tests.policy b/gradle/testing/policies/tests.policy
new file mode 100644
index 0000000..9cdfaa1
--- /dev/null
+++ b/gradle/testing/policies/tests.policy
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Policy file for lucene tests. Please keep minimal and avoid wildcards.
+
+grant {
+ // 3rd party jar resources (where symlinks are not supported), test-files/ resources
+ permission java.io.FilePermission "${common.dir}${/}-", "read";
+ // system jar resources, and let TestIndexWriterOnJRECrash fork its jvm
+ permission java.io.FilePermission "${java.home}${/}-", "read,execute";
+
+ // write only to sandbox
+ permission java.io.FilePermission "${java.io.tmpdir}", "read,write";
+ permission java.io.FilePermission "${java.io.tmpdir}${/}-", "read,write,delete";
+ permission java.io.FilePermission "${tests.linedocsfile}", "read";
+
+ // misc HardlinkCopyDirectoryWrapper needs this to test if hardlinks can be created
+ permission java.nio.file.LinkPermission "hard";
+ // needed by SSD detection tests in TestIOUtils (creates symlinks)
+ permission java.nio.file.LinkPermission "symbolic";
+
+// needed by gson serialization of junit4 runner: TODO clean that up
+ permission java.lang.RuntimePermission "accessDeclaredMembers";
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ // needed by junit4 runner to capture sysout/syserr:
+ permission java.lang.RuntimePermission "setIO";
+ // needed by randomized runner to catch failures from other threads:
+ permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler";
+ // needed by randomized runner getTopThreadGroup:
+ permission java.lang.RuntimePermission "modifyThreadGroup";
+ // needed by tests e.g. shutting down executors:
+ permission java.lang.RuntimePermission "modifyThread";
+ // needed for tons of test hacks etc
+ permission java.lang.RuntimePermission "getStackTrace";
+ // needed for mock filesystems in tests
+ permission java.lang.RuntimePermission "fileSystemProvider";
+ // needed for test of IOUtils.spins (maybe it can be avoided)
+ permission java.lang.RuntimePermission "getFileStoreAttributes";
+ // analyzers/uima: needed by lucene expressions' JavascriptCompiler
+ permission java.lang.RuntimePermission "createClassLoader";
+ // needed to test unmap hack on platforms that support it
+ permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
+ // needed by cyberneko usage by benchmarks on J9
+ permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util";
+ // needed by jacoco to dump coverage
+ permission java.lang.RuntimePermission "shutdownHooks";
+ // needed by org.apache.logging.log4j
+ permission java.lang.RuntimePermission "getenv.*";
+ permission java.lang.RuntimePermission "getClassLoader";
+ permission java.lang.RuntimePermission "setContextClassLoader";
+
+ // read access to all system properties:
+ permission java.util.PropertyPermission "*", "read";
+ // write access to only these:
+ // locale randomization
+ permission java.util.PropertyPermission "user.language", "write";
+ // timezone randomization
+ permission java.util.PropertyPermission "user.timezone", "write";
+
+ // CMS randomization
+ permission java.util.PropertyPermission "lucene.cms.override_core_count", "write";
+ permission java.util.PropertyPermission "lucene.cms.override_spins", "write";
+
+ // used by nested tests? (e.g. TestLeaveFilesIfTestFails). TODO: look into this
+ permission java.util.PropertyPermission "tests.runnested", "write";
+
+ // solr properties. TODO: move these out to SolrTestCase
+ permission java.util.PropertyPermission "solr.data.dir", "write";
+ permission java.util.PropertyPermission "solr.solr.home", "write";
+ permission java.util.PropertyPermission "solr.directoryFactory", "write";
+
+ // replicator: jetty tests require some network permissions:
+ // all possibilities of accepting/binding/connecting on localhost with ports >= 1024:
+ permission java.net.SocketPermission "localhost:1024-", "accept,listen,connect,resolve";
+ permission java.net.SocketPermission "127.0.0.1:1024-", "accept,listen,connect,resolve";
+ permission java.net.SocketPermission "[::1]:1024-", "accept,listen,connect,resolve";
+
+ // SSL related properties for jetty
+ permission java.security.SecurityPermission "getProperty.ssl.KeyManagerFactory.algorithm";
+ permission java.security.SecurityPermission "getProperty.ssl.TrustManagerFactory.algorithm";
+
+ // allows LuceneTestCase#runWithRestrictedPermissions to execute with lower (or no) permission
+ permission java.security.SecurityPermission "createAccessControlContext";
+};
+
+
+// Grant all permissions to Gradle test runner classes.
+
+grant codeBase "file:${gradle.lib.dir}${/}-" {
+ permission java.security.AllPermission;
+};
+
+grant codeBase "file:${gradle.worker.jar}" {
+ permission java.security.AllPermission;
+};
+
+grant {
+ // Allow reading gradle worker JAR.
+ permission java.io.FilePermission "${gradle.worker.jar}", "read";
+ // Allow reading from classpath JARs (resources).
+ permission java.io.FilePermission "${gradle.user.home}${/}-", "read";
+};
diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradle
index b97ad7f..6df7ce6 100644
--- a/gradle/testing/randomization.gradle
+++ b/gradle/testing/randomization.gradle
@@ -2,6 +2,7 @@
// Configure test randomization seeds and derived test properties.
//
+import java.nio.file.*
import org.apache.tools.ant.taskdefs.condition.Os
import com.carrotsearch.randomizedtesting.SeedUtils
import com.carrotsearch.randomizedtesting.generators.RandomPicks
@@ -58,11 +59,13 @@ allprojects {
[propName: 'tests.multiplier', value: 1, description: "Value multiplier for randomized tests."],
[propName: 'tests.maxfailures', value: null, description: "Skip tests after a given number of failures."],
[propName: 'tests.timeoutSuite', value: null, description: "Timeout (in millis) for an entire suite."],
+ [propName: 'tests.failfast', value: "false", description: "Stop the build early on failure.", buildOnly: true],
// asserts, debug output.
[propName: 'tests.asserts', value: "true", description: "Enables or disables assertions mode."],
[propName: 'tests.verbose', value: false, description: "Emit verbose debug information from tests."],
[propName: 'tests.infostream', value: null, description: "Enables or disables infostream logs."],
[propName: 'tests.leaveTemporary', value: null, description: "Leave temporary directories after tests complete."],
+ [propName: 'tests.useSecurityManager', value: false, description: "Enable security manager in tests.", buildOnly: true],
// component randomization
[propName: 'tests.codec', value: "random", description: "Sets the codec tests should run with."],
[propName: 'tests.directory', value: "random", description: "Sets the Directory implementation tests should run with."],
@@ -77,7 +80,7 @@ allprojects {
[propName: 'tests.weekly', value: false, description: "Enables or disables @Weekly tests."],
[propName: 'tests.monster', value: false, description: "Enables or disables @Monster tests."],
[propName: 'tests.awaitsfix', value: null, description: "Enables or disables @AwaitsFix tests."],
- [propName: 'tests.file.encoding', value: "random", description: "Sets the default file.encoding on test JVM."],
+ [propName: 'tests.file.encoding', value: "random", description: "Sets the default file.encoding on test JVM.", buildOnly: true],
// test data
[propName: 'tests.linedocsfile', value: 'europarl.lines.txt.gz', description: "Test data file path."],
// miscellaneous; some of them very weird.
@@ -111,7 +114,7 @@ allprojects {
ext.testOptionsResolved = testOptions.findAll { opt ->
propertyOrDefault(opt.propName, opt.value) != null
}.collectEntries { opt ->
- [(opt.propName): propertyOrDefault(opt.propName, opt.value)]
+ [(opt.propName): Objects.toString(propertyOrDefault(opt.propName, opt.value))]
}
// These are not official options or dynamically seed-derived options.
@@ -132,11 +135,12 @@ allprojects {
].find { prop ->
Boolean.parseBoolean(propertyOrDefault(prop, "false"))
}) {
- testOptionsResolved['tests.leaveTemporary'] = true
+ testOptionsResolved['tests.leaveTemporary'] = "true"
}
// Append resolved test properties to the test task.
test {
+ // TODO: we could remove opts with "buildOnly: true" (?)
systemProperties testOptionsResolved
if (Boolean.parseBoolean(testOptionsResolved['tests.asserts'])) {
@@ -145,6 +149,38 @@ allprojects {
enableAssertions = false
}
+ if (Boolean.parseBoolean(testOptionsResolved["tests.failfast"])) {
+ failFast true
+ }
+
+ // Enable security manager, if requested. We could move the selection of security manager and security policy
+ // to each project's build/ configuration but it seems compact enough to keep it here for now.
+ if (Boolean.parseBoolean(testOptionsResolved["tests.useSecurityManager"])) {
+ if (project.path == ":lucene:replicator") {
+ // systemProperty 'java.security.manager', "org.apache.lucene.util.TestSecurityManager"
+ // systemProperty 'java.security.policy', file("${commonDir}/tools/junit4/replicator-tests.policy")
+ doFirst {
+ logger.log(LogLevel.WARN, "Running with security manager not implemented for: ${project.path}")
+ }
+ } else if (project.path.startsWith(":lucene")) {
+ systemProperty 'java.security.manager', "org.apache.lucene.util.TestSecurityManager"
+ systemProperty 'java.security.policy', rootProject.file("gradle/testing/policies/tests.policy")
+ } else {
+ // systemProperty 'java.security.manager', "org.apache.solr.util.SolrSecurityManager"
+ // systemProperty 'java.security.policy', file("${commonDir}/tools/junit4/solr-tests.policy")
+ doFirst {
+ logger.log(LogLevel.WARN, "Running with security manager not implemented for: ${project.path}")
+ }
+ }
+
+ systemProperty 'common.dir', commonDir
+
+ def gradleUserHome = project.gradle.getGradleUserHomeDir()
+ systemProperty 'gradle.lib.dir', Paths.get(project.class.location.toURI()).parent.toAbsolutePath().toString().replace('\\', '/')
+ systemProperty 'gradle.worker.jar', Paths.get("${gradleUserHome}/caches/${gradle.gradleVersion}/workerMain/gradle-worker.jar").toAbsolutePath().toString()
+ systemProperty 'gradle.user.home', gradleUserHome.toPath().toAbsolutePath().toString()
+ }
+
doFirst {
logger.debug("Will use test opts:\n" + testOptionsResolved.collect {k,v -> "${k}: ${v}"}.sort().join("\n"))
}
[lucene-solr] 01/02: Merge remote-tracking branch 'origin/master'
into gradle-master
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch gradle-master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit daa0779ff411e427126b374ae00427ced3800872
Merge: de8a37e 9ec8a86
Author: Dawid Weiss <dw...@apache.org>
AuthorDate: Fri Dec 6 16:32:15 2019 +0100
Merge remote-tracking branch 'origin/master' into gradle-master
.../src/java/org/apache/lucene/util/TestSecurityManager.java | 4 +++-
lucene/tools/junit4/solr-tests.policy | 1 +
2 files changed, 4 insertions(+), 1 deletion(-)