You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/04/18 18:35:50 UTC
svn commit: r1674537 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/test-framework/
lucene/test-framework/src/java/org/apache/lucene/util/ lucene/tools/
lucene/tools/junit4/
Author: mikemccand
Date: Sat Apr 18 16:35:49 2015
New Revision: 1674537
URL: http://svn.apache.org/r1674537
Log:
LUCENE-6437: randomize cpu core count and spins for ConcurrentMergeScheduler's dynamic defaults
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/branch_5x/lucene/tools/ (props changed)
lucene/dev/branches/branch_5x/lucene/tools/junit4/tests.policy
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1674537&r1=1674536&r2=1674537&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Sat Apr 18 16:35:49 2015
@@ -117,6 +117,11 @@ Test Framework
* LUCENE-6419: Added two-phase iteration assertions to AssertingQuery.
(Adrien Grand)
+* LUCENE-6437: Randomly set CPU core count and spins, derived from
+ test's master seed, used by ConcurrentMergeScheduler to set dynamic
+ defaults, for better test randomization and to help tests reproduce
+ (Robert Muir, Mike McCandless)
+
======================= Lucene 5.1.0 =======================
New Features
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1674537&r1=1674536&r2=1674537&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Sat Apr 18 16:35:49 2015
@@ -67,6 +67,16 @@ public class ConcurrentMergeScheduler ex
* Linux; other platforms will assume the index is not on an SSD. */
public static final int AUTO_DETECT_MERGES_AND_THREADS = -1;
+ /** Used for testing.
+ *
+ * @lucene.internal */
+ public static final String DEFAULT_CPU_CORE_COUNT_PROPERTY = "lucene.cms.override_core_count";
+
+ /** Used for testing.
+ *
+ * @lucene.internal */
+ public static final String DEFAULT_SPINS_PROPERTY = "lucene.cms.override_spins";
+
/** List of currently active {@link MergeThread}s. */
protected final List<MergeThread> mergeThreads = new ArrayList<>();
@@ -159,7 +169,19 @@ public class ConcurrentMergeScheduler ex
maxThreadCount = 1;
maxMergeCount = 6;
} else {
- maxThreadCount = Math.max(1, Math.min(4, Runtime.getRuntime().availableProcessors()/2));
+ int coreCount = Runtime.getRuntime().availableProcessors();
+
+ // Let tests override this to help reproducing a failure on a machine that has a different
+ // core count than the one where the test originally failed:
+ try {
+ String value = System.getProperty(DEFAULT_CPU_CORE_COUNT_PROPERTY);
+ if (value != null) {
+ coreCount = Integer.parseInt(value);
+ }
+ } catch (Throwable ignored) {
+ }
+
+ maxThreadCount = Math.max(1, Math.min(4, coreCount/2));
maxMergeCount = maxThreadCount+5;
}
}
@@ -347,6 +369,16 @@ public class ConcurrentMergeScheduler ex
private synchronized void initDynamicDefaults(IndexWriter writer) throws IOException {
if (maxThreadCount == AUTO_DETECT_MERGES_AND_THREADS) {
boolean spins = IOUtils.spins(writer.getDirectory());
+
+ // Let tests override this to help reproducing a failure on a machine that has a different
+ // core count than the one where the test originally failed:
+ try {
+ String value = System.getProperty(DEFAULT_SPINS_PROPERTY);
+ if (value != null) {
+ spins = Boolean.parseBoolean(value);
+ }
+ } catch (Throwable ignored) {
+ }
setDefaultMaxMergesAndThreads(spins);
if (verbose()) {
message("initDynamicDefaults spins=" + spins + " maxThreadCount=" + maxThreadCount + " maxMergeCount=" + maxMergeCount);
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1674537&r1=1674536&r2=1674537&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Sat Apr 18 16:35:49 2015
@@ -1703,6 +1703,32 @@ public abstract class LuceneTestCase ext
IndexSearcher.setDefaultQueryCachingPolicy(DEFAULT_CACHING_POLICY);
}
+ @BeforeClass
+ public static void setupCPUCoreCount() {
+ // Randomize core count so CMS varies its dynamic defaults, and this also "fixes" core
+ // count from the master seed so it will always be the same on reproduce:
+ int numCores = TestUtil.nextInt(random(), 1, 4);
+ System.setProperty(ConcurrentMergeScheduler.DEFAULT_CPU_CORE_COUNT_PROPERTY, Integer.toString(numCores));
+ }
+
+ @AfterClass
+ public static void restoreCPUCoreCount() {
+ System.clearProperty(ConcurrentMergeScheduler.DEFAULT_CPU_CORE_COUNT_PROPERTY);
+ }
+
+ @BeforeClass
+ public static void setupSpins() {
+ // Randomize IOUtils.spins() count so CMS varies its dynamic defaults, and this also "fixes" core
+ // count from the master seed so it will always be the same on reproduce:
+ boolean spins = random().nextBoolean();
+ System.setProperty(ConcurrentMergeScheduler.DEFAULT_SPINS_PROPERTY, Boolean.toString(spins));
+ }
+
+ @AfterClass
+ public static void restoreSpins() {
+ System.clearProperty(ConcurrentMergeScheduler.DEFAULT_SPINS_PROPERTY);
+ }
+
/**
* Create a new searcher over the reader. This searcher might randomly use
* threads.
Modified: lucene/dev/branches/branch_5x/lucene/tools/junit4/tests.policy
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/tools/junit4/tests.policy?rev=1674537&r1=1674536&r2=1674537&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/tools/junit4/tests.policy (original)
+++ lucene/dev/branches/branch_5x/lucene/tools/junit4/tests.policy Sat Apr 18 16:35:49 2015
@@ -84,6 +84,10 @@ grant {
// timezone randomization
permission java.util.PropertyPermission "user.timezone", "write";
+ // CMS randomization
+ permission java.util.PropertyPermission "lucene.cms.override_core_count", "write";
+ permission java.util.PropertyPermission "lucene.cms.override_spins", "write";
+
// used by nested tests? (e.g. TestLeaveFilesIfTestFails). TODO: look into this
permission java.util.PropertyPermission "tests.runnested", "write";