You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2012/07/04 18:02:42 UTC
svn commit: r1357324 [2/2] - in
/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search:
TestRTGBase.java TestRealTimeGet.java TestStressLucene.java
TestStressRecovery.java TestStressReorder.java TestStressVersions.java
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java?rev=1357324&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java Wed Jul 4 16:02:42 2012
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+
+import org.apache.lucene.util.Constants;
+import org.apache.noggit.ObjectBuilder;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.update.UpdateHandler;
+import org.apache.solr.update.UpdateLog;
+import org.apache.solr.update.VersionInfo;
+import org.apache.solr.util.TestHarness;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.apache.solr.core.SolrCore.verbose;
+import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+
+public class TestStressRecovery extends TestRTGBase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-tlog.xml","schema15.xml");
+ }
+
+
+ // This points to the live model when state is ACTIVE, but a snapshot of the
+ // past when recovering.
+ volatile ConcurrentHashMap<Integer,DocInfo> visibleModel;
+
+ // This version simulates updates coming from the leader and sometimes being reordered
+ // and tests the ability to buffer updates and apply them later
+ @Test
+ public void testStressRecovery() throws Exception {
+ assumeFalse("FIXME: This test is horribly slow sometimes on Windows!", Constants.WINDOWS);
+ clearIndex();
+ assertU(commit());
+
+ final int commitPercent = 5 + random().nextInt(10);
+ final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft
+ final int deletePercent = 4+random().nextInt(25);
+ final int deleteByQueryPercent = random().nextInt(5);
+ final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
+ int nWriteThreads = 2 + random().nextInt(10); // fewer write threads to give recovery thread more of a chance
+
+ final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
+
+ // query variables
+ final int percentRealtimeQuery = 75;
+ final int percentGetLatestVersions = random().nextInt(4);
+ final AtomicLong operations = new AtomicLong(atLeast(100)); // number of recovery loops to perform
+ int nReadThreads = 2 + random().nextInt(10); // fewer read threads to give writers more of a chance
+
+ initModel(ndocs);
+
+ final AtomicInteger numCommitting = new AtomicInteger();
+
+ List<Thread> threads = new ArrayList<Thread>();
+
+
+ final AtomicLong testVersion = new AtomicLong(0);
+
+
+ final UpdateHandler uHandler = h.getCore().getUpdateHandler();
+ final UpdateLog uLog = uHandler.getUpdateLog();
+ final VersionInfo vInfo = uLog.getVersionInfo();
+ final Object stateChangeLock = new Object();
+ this.visibleModel = model;
+ final Semaphore[] writePermissions = new Semaphore[nWriteThreads];
+ for (int i=0; i<nWriteThreads; i++) writePermissions[i] = new Semaphore(Integer.MAX_VALUE, false);
+
+ final Semaphore readPermission = new Semaphore(Integer.MAX_VALUE, false);
+
+ for (int i=0; i<nWriteThreads; i++) {
+ final int threadNum = i;
+
+ Thread thread = new Thread("WRITER"+i) {
+ Random rand = new Random(random().nextInt());
+ Semaphore writePermission = writePermissions[threadNum];
+
+ @Override
+ public void run() {
+ try {
+ while (operations.get() > 0) {
+ writePermission.acquire();
+
+ int oper = rand.nextInt(10);
+
+ if (oper < commitPercent) {
+ if (numCommitting.incrementAndGet() <= maxConcurrentCommits) {
+ Map<Integer,DocInfo> newCommittedModel;
+ long version;
+
+ synchronized(globalLock) {
+ newCommittedModel = new HashMap<Integer,DocInfo>(model); // take a snapshot
+ version = snapshotCount++;
+ }
+
+ synchronized (stateChangeLock) {
+ // These commits won't take affect if we are in recovery mode,
+ // so change the version to -1 so we won't update our model.
+ if (uLog.getState() != UpdateLog.State.ACTIVE) version = -1;
+ if (rand.nextInt(100) < softCommitPercent) {
+ verbose("softCommit start");
+ assertU(TestHarness.commit("softCommit","true"));
+ verbose("softCommit end");
+ } else {
+ verbose("hardCommit start");
+ assertU(commit());
+ verbose("hardCommit end");
+ }
+ }
+
+ synchronized(globalLock) {
+ // install this model snapshot only if it's newer than the current one
+ // install this model only if we are not in recovery mode.
+ if (version >= committedModelClock) {
+ if (VERBOSE) {
+ verbose("installing new committedModel version="+committedModelClock);
+ }
+ committedModel = newCommittedModel;
+ committedModelClock = version;
+ }
+ }
+ }
+ numCommitting.decrementAndGet();
+ continue;
+ }
+
+
+ int id;
+
+ if (rand.nextBoolean()) {
+ id = rand.nextInt(ndocs);
+ } else {
+ id = lastId; // reuse the last ID half of the time to force more race conditions
+ }
+
+ // set the lastId before we actually change it sometimes to try and
+ // uncover more race conditions between writing and reading
+ boolean before = rand.nextBoolean();
+ if (before) {
+ lastId = id;
+ }
+
+ DocInfo info = model.get(id);
+
+ long val = info.val;
+ long nextVal = Math.abs(val)+1;
+
+ // the version we set on the update should determine who wins
+ // These versions are not derived from the actual leader update handler hand hence this
+ // test may need to change depending on how we handle version numbers.
+ long version = testVersion.incrementAndGet();
+
+ // yield after getting the next version to increase the odds of updates happening out of order
+ if (rand.nextBoolean()) Thread.yield();
+
+ if (oper < commitPercent + deletePercent) {
+ verbose("deleting id",id,"val=",nextVal,"version",version);
+
+ Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER));
+
+ // TODO: returning versions for these types of updates is redundant
+ // but if we do return, they had better be equal
+ if (returnedVersion != null) {
+ assertEquals(-version, returnedVersion.longValue());
+ }
+
+ // only update model if the version is newer
+ synchronized (model) {
+ DocInfo currInfo = model.get(id);
+ if (Math.abs(version) > Math.abs(currInfo.version)) {
+ model.put(id, new DocInfo(version, -nextVal));
+ }
+ }
+
+ verbose("deleting id", id, "val=",nextVal,"version",version,"DONE");
+ } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) {
+
+ verbose("deleteByQuery id",id,"val=",nextVal,"version",version);
+
+ Long returnedVersion = deleteByQueryAndGetVersion("id:"+Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER));
+
+ // TODO: returning versions for these types of updates is redundant
+ // but if we do return, they had better be equal
+ if (returnedVersion != null) {
+ assertEquals(-version, returnedVersion.longValue());
+ }
+
+ // only update model if the version is newer
+ synchronized (model) {
+ DocInfo currInfo = model.get(id);
+ if (Math.abs(version) > Math.abs(currInfo.version)) {
+ model.put(id, new DocInfo(version, -nextVal));
+ }
+ }
+
+ verbose("deleteByQuery id", id, "val=",nextVal,"version",version,"DONE");
+
+ } else {
+ verbose("adding id", id, "val=", nextVal,"version",version);
+
+ Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), field, Long.toString(nextVal), "_version_",Long.toString(version)), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ if (returnedVersion != null) {
+ assertEquals(version, returnedVersion.longValue());
+ }
+
+ // only update model if the version is newer
+ synchronized (model) {
+ DocInfo currInfo = model.get(id);
+ if (version > currInfo.version) {
+ model.put(id, new DocInfo(version, nextVal));
+ }
+ }
+
+ if (VERBOSE) {
+ verbose("adding id", id, "val=", nextVal,"version",version,"DONE");
+ }
+
+ }
+ // } // end sync
+
+ if (!before) {
+ lastId = id;
+ }
+ }
+ } catch (Throwable e) {
+ operations.set(-1L);
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ threads.add(thread);
+ }
+
+
+ for (int i=0; i<nReadThreads; i++) {
+ Thread thread = new Thread("READER"+i) {
+ Random rand = new Random(random().nextInt());
+
+ @Override
+ public void run() {
+ try {
+ while (operations.get() > 0) {
+ // throttle reads (don't completely stop)
+ readPermission.tryAcquire(10, TimeUnit.MILLISECONDS);
+
+
+ // bias toward a recently changed doc
+ int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs);
+
+ // when indexing, we update the index, then the model
+ // so when querying, we should first check the model, and then the index
+
+ boolean realTime = rand.nextInt(100) < percentRealtimeQuery;
+ DocInfo info;
+
+ if (realTime) {
+ info = visibleModel.get(id);
+ } else {
+ synchronized(globalLock) {
+ info = committedModel.get(id);
+ }
+ }
+
+
+ if (VERBOSE) {
+ verbose("querying id", id);
+ }
+ SolrQueryRequest sreq;
+ if (realTime) {
+ sreq = req("wt","json", "qt","/get", "ids",Integer.toString(id));
+ } else {
+ sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true");
+ }
+
+ String response = h.query(sreq);
+ Map rsp = (Map)ObjectBuilder.fromJSON(response);
+ List doclist = (List)(((Map)rsp.get("response")).get("docs"));
+ if (doclist.size() == 0) {
+ // there's no info we can get back with a delete, so not much we can check without further synchronization
+ } else {
+ assertEquals(1, doclist.size());
+ long foundVal = (Long)(((Map)doclist.get(0)).get(field));
+ long foundVer = (Long)(((Map)doclist.get(0)).get("_version_"));
+ if (foundVer < Math.abs(info.version)
+ || (foundVer == info.version && foundVal != info.val) ) { // if the version matches, the val must
+ verbose("ERROR, id=", id, "found=",response,"model",info);
+ assertTrue(false);
+ }
+ }
+ }
+
+
+ if (rand.nextInt(100) < percentGetLatestVersions) {
+ getLatestVersions();
+ // TODO: some sort of validation that the latest version is >= to the latest version we added?
+ }
+
+ } catch (Throwable e) {
+ operations.set(-1L);
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ threads.add(thread);
+ }
+
+
+ for (Thread thread : threads) {
+ thread.start();
+ }
+
+ int bufferedAddsApplied = 0;
+ do {
+ assertTrue(uLog.getState() == UpdateLog.State.ACTIVE);
+
+ // before we start buffering updates, we want to point
+ // visibleModel away from the live model.
+
+ visibleModel = new ConcurrentHashMap<Integer, DocInfo>(model);
+
+ synchronized (stateChangeLock) {
+ uLog.bufferUpdates();
+ }
+
+ assertTrue(uLog.getState() == UpdateLog.State.BUFFERING);
+
+ // sometimes wait for a second to allow time for writers to write something
+ if (random().nextBoolean()) Thread.sleep(random().nextInt(10)+1);
+
+ Future<UpdateLog.RecoveryInfo> recoveryInfoF = uLog.applyBufferedUpdates();
+ if (recoveryInfoF != null) {
+ UpdateLog.RecoveryInfo recInfo = null;
+
+ int writeThreadNumber = 0;
+ while (recInfo == null) {
+ try {
+ // wait a short period of time for recovery to complete (and to give a chance for more writers to concurrently add docs)
+ recInfo = recoveryInfoF.get(random().nextInt(100/nWriteThreads), TimeUnit.MILLISECONDS);
+ } catch (TimeoutException e) {
+ // idle one more write thread
+ verbose("Operation",operations.get(),"Draining permits for write thread",writeThreadNumber);
+ writePermissions[writeThreadNumber++].drainPermits();
+ if (writeThreadNumber >= nWriteThreads) {
+ // if we hit the end, back up and give a few write permits
+ writeThreadNumber--;
+ writePermissions[writeThreadNumber].release(random().nextInt(2) + 1);
+ }
+
+ // throttle readers so they don't steal too much CPU from the recovery thread
+ readPermission.drainPermits();
+ }
+ }
+
+ bufferedAddsApplied += recInfo.adds;
+ }
+
+ // put all writers back at full blast
+ for (Semaphore writePerm : writePermissions) {
+ // I don't think semaphores check for overflow, so we need to check mow many remain
+ int neededPermits = Integer.MAX_VALUE - writePerm.availablePermits();
+ if (neededPermits > 0) writePerm.release( neededPermits );
+ }
+
+ // put back readers at full blast and point back to live model
+ visibleModel = model;
+ int neededPermits = Integer.MAX_VALUE - readPermission.availablePermits();
+ if (neededPermits > 0) readPermission.release( neededPermits );
+
+ verbose("ROUND=",operations.get());
+ } while (operations.decrementAndGet() > 0);
+
+ verbose("bufferedAddsApplied=",bufferedAddsApplied);
+
+ for (Thread thread : threads) {
+ thread.join();
+ }
+
+ }
+
+}
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressReorder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressReorder.java?rev=1357324&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressReorder.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressReorder.java Wed Jul 4 16:02:42 2012
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+
+import org.apache.noggit.ObjectBuilder;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.util.TestHarness;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.apache.solr.core.SolrCore.verbose;
+import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+
+public class TestStressReorder extends TestRTGBase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-tlog.xml","schema15.xml");
+ }
+
+
+ // This version simulates updates coming from the leader and sometimes being reordered
+ @Test
+ public void testStressReorderVersions() throws Exception {
+ clearIndex();
+ assertU(commit());
+
+ final int commitPercent = 5 + random().nextInt(20);
+ final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft
+ final int deletePercent = 4+random().nextInt(25);
+ final int deleteByQueryPercent = 0; // 1+random().nextInt(7);
+ final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
+ int nWriteThreads = 5 + random().nextInt(25);
+
+ final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
+
+ // query variables
+ final int percentRealtimeQuery = 75;
+ final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total
+ int nReadThreads = 5 + random().nextInt(25);
+
+
+ /** // testing
+ final int commitPercent = 5;
+ final int softCommitPercent = 100; // what percent of the commits are soft
+ final int deletePercent = 0;
+ final int deleteByQueryPercent = 50;
+ final int ndocs = 1;
+ int nWriteThreads = 2;
+
+ final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
+
+ // query variables
+ final int percentRealtimeQuery = 101;
+ final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total
+ int nReadThreads = 1;
+ **/
+
+
+ initModel(ndocs);
+
+ final AtomicInteger numCommitting = new AtomicInteger();
+
+ List<Thread> threads = new ArrayList<Thread>();
+
+
+ final AtomicLong testVersion = new AtomicLong(0);
+
+ for (int i=0; i<nWriteThreads; i++) {
+ Thread thread = new Thread("WRITER"+i) {
+ Random rand = new Random(random().nextInt());
+
+ @Override
+ public void run() {
+ try {
+ while (operations.get() > 0) {
+ int oper = rand.nextInt(100);
+
+ if (oper < commitPercent) {
+ if (numCommitting.incrementAndGet() <= maxConcurrentCommits) {
+ Map<Integer,DocInfo> newCommittedModel;
+ long version;
+
+ synchronized(TestStressReorder.this) {
+ newCommittedModel = new HashMap<Integer,DocInfo>(model); // take a snapshot
+ version = snapshotCount++;
+ }
+
+ if (rand.nextInt(100) < softCommitPercent) {
+ verbose("softCommit start");
+ assertU(TestHarness.commit("softCommit","true"));
+ verbose("softCommit end");
+ } else {
+ verbose("hardCommit start");
+ assertU(commit());
+ verbose("hardCommit end");
+ }
+
+ synchronized(TestStressReorder.this) {
+ // install this model snapshot only if it's newer than the current one
+ if (version >= committedModelClock) {
+ if (VERBOSE) {
+ verbose("installing new committedModel version="+committedModelClock);
+ }
+ committedModel = newCommittedModel;
+ committedModelClock = version;
+ }
+ }
+ }
+ numCommitting.decrementAndGet();
+ continue;
+ }
+
+
+ int id;
+
+ if (rand.nextBoolean()) {
+ id = rand.nextInt(ndocs);
+ } else {
+ id = lastId; // reuse the last ID half of the time to force more race conditions
+ }
+
+ // set the lastId before we actually change it sometimes to try and
+ // uncover more race conditions between writing and reading
+ boolean before = rand.nextBoolean();
+ if (before) {
+ lastId = id;
+ }
+
+ DocInfo info = model.get(id);
+
+ long val = info.val;
+ long nextVal = Math.abs(val)+1;
+
+ // the version we set on the update should determine who wins
+ // These versions are not derived from the actual leader update handler hand hence this
+ // test may need to change depending on how we handle version numbers.
+ long version = testVersion.incrementAndGet();
+
+ // yield after getting the next version to increase the odds of updates happening out of order
+ if (rand.nextBoolean()) Thread.yield();
+
+ if (oper < commitPercent + deletePercent) {
+ verbose("deleting id",id,"val=",nextVal,"version",version);
+
+ Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER));
+
+ // TODO: returning versions for these types of updates is redundant
+ // but if we do return, they had better be equal
+ if (returnedVersion != null) {
+ assertEquals(-version, returnedVersion.longValue());
+ }
+
+ // only update model if the version is newer
+ synchronized (model) {
+ DocInfo currInfo = model.get(id);
+ if (Math.abs(version) > Math.abs(currInfo.version)) {
+ model.put(id, new DocInfo(version, -nextVal));
+ }
+ }
+
+ verbose("deleting id", id, "val=",nextVal,"version",version,"DONE");
+ } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) {
+
+ verbose("deleteByQuery id",id,"val=",nextVal,"version",version);
+
+ Long returnedVersion = deleteByQueryAndGetVersion("id:"+Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER));
+
+ // TODO: returning versions for these types of updates is redundant
+ // but if we do return, they had better be equal
+ if (returnedVersion != null) {
+ assertEquals(-version, returnedVersion.longValue());
+ }
+
+ // only update model if the version is newer
+ synchronized (model) {
+ DocInfo currInfo = model.get(id);
+ if (Math.abs(version) > Math.abs(currInfo.version)) {
+ model.put(id, new DocInfo(version, -nextVal));
+ }
+ }
+
+ verbose("deleteByQuery id", id, "val=",nextVal,"version",version,"DONE");
+
+ } else {
+ verbose("adding id", id, "val=", nextVal,"version",version);
+
+ Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), field, Long.toString(nextVal), "_version_",Long.toString(version)), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ if (returnedVersion != null) {
+ assertEquals(version, returnedVersion.longValue());
+ }
+
+ // only update model if the version is newer
+ synchronized (model) {
+ DocInfo currInfo = model.get(id);
+ if (version > currInfo.version) {
+ model.put(id, new DocInfo(version, nextVal));
+ }
+ }
+
+ if (VERBOSE) {
+ verbose("adding id", id, "val=", nextVal,"version",version,"DONE");
+ }
+
+ }
+ // } // end sync
+
+ if (!before) {
+ lastId = id;
+ }
+ }
+ } catch (Throwable e) {
+ operations.set(-1L);
+ log.error("",e);
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ threads.add(thread);
+ }
+
+
+ for (int i=0; i<nReadThreads; i++) {
+ Thread thread = new Thread("READER"+i) {
+ Random rand = new Random(random().nextInt());
+
+ @Override
+ public void run() {
+ try {
+ while (operations.decrementAndGet() >= 0) {
+ // bias toward a recently changed doc
+ int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs);
+
+ // when indexing, we update the index, then the model
+ // so when querying, we should first check the model, and then the index
+
+ boolean realTime = rand.nextInt(100) < percentRealtimeQuery;
+ DocInfo info;
+
+ if (realTime) {
+ info = model.get(id);
+ } else {
+ synchronized(TestStressReorder.this) {
+ info = committedModel.get(id);
+ }
+ }
+
+ if (VERBOSE) {
+ verbose("querying id", id);
+ }
+ SolrQueryRequest sreq;
+ if (realTime) {
+ sreq = req("wt","json", "qt","/get", "ids",Integer.toString(id));
+ } else {
+ sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true");
+ }
+
+ String response = h.query(sreq);
+ Map rsp = (Map)ObjectBuilder.fromJSON(response);
+ List doclist = (List)(((Map)rsp.get("response")).get("docs"));
+ if (doclist.size() == 0) {
+ // there's no info we can get back with a delete, so not much we can check without further synchronization
+ } else {
+ assertEquals(1, doclist.size());
+ long foundVal = (Long)(((Map)doclist.get(0)).get(field));
+ long foundVer = (Long)(((Map)doclist.get(0)).get("_version_"));
+ if (foundVer < Math.abs(info.version)
+ || (foundVer == info.version && foundVal != info.val) ) { // if the version matches, the val must
+ log.error("ERROR, id=" + id + " found=" + response + " model" + info);
+ assertTrue(false);
+ }
+ }
+ }
+ } catch (Throwable e) {
+ operations.set(-1L);
+ log.error("",e);
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ threads.add(thread);
+ }
+
+
+ for (Thread thread : threads) {
+ thread.start();
+ }
+
+ for (Thread thread : threads) {
+ thread.join();
+ }
+
+ }
+
+}
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressVersions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressVersions.java?rev=1357324&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressVersions.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressVersions.java Wed Jul 4 16:02:42 2012
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+
+import org.apache.noggit.ObjectBuilder;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.util.TestHarness;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.apache.solr.core.SolrCore.verbose;
+
+public class TestStressVersions extends TestRTGBase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-tlog.xml","schema15.xml");
+ }
+
+
+ // This version doesn't synchronize on id to tell what update won, but instead uses versions
+ @Test
+ public void testStressGetRealtimeVersions() throws Exception {
+ clearIndex();
+ assertU(commit());
+
+ final int commitPercent = 5 + random().nextInt(20);
+ final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft
+ final int deletePercent = 4+random().nextInt(25);
+ final int deleteByQueryPercent = 1 + random().nextInt(5);
+ final int optimisticPercent = 1+random().nextInt(50); // percent change that an update uses optimistic locking
+ final int optimisticCorrectPercent = 25+random().nextInt(70); // percent change that a version specified will be correct
+ final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
+ int nWriteThreads = 5 + random().nextInt(25);
+
+ final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
+
+ // query variables
+ final int percentRealtimeQuery = 75;
+ final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total
+ int nReadThreads = 5 + random().nextInt(25);
+
+
+
+ initModel(ndocs);
+
+ final AtomicInteger numCommitting = new AtomicInteger();
+
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for (int i=0; i<nWriteThreads; i++) {
+ Thread thread = new Thread("WRITER"+i) {
+ Random rand = new Random(random().nextInt());
+
+ @Override
+ public void run() {
+ try {
+ while (operations.get() > 0) {
+ int oper = rand.nextInt(100);
+
+ if (oper < commitPercent) {
+ if (numCommitting.incrementAndGet() <= maxConcurrentCommits) {
+ Map<Integer,DocInfo> newCommittedModel;
+ long version;
+
+ synchronized(globalLock) {
+ newCommittedModel = new HashMap<Integer,DocInfo>(model); // take a snapshot
+ version = snapshotCount++;
+ }
+
+ if (rand.nextInt(100) < softCommitPercent) {
+ verbose("softCommit start");
+ assertU(TestHarness.commit("softCommit","true"));
+ verbose("softCommit end");
+ } else {
+ verbose("hardCommit start");
+ assertU(commit());
+ verbose("hardCommit end");
+ }
+
+ synchronized(globalLock) {
+ // install this model snapshot only if it's newer than the current one
+ if (version >= committedModelClock) {
+ if (VERBOSE) {
+ verbose("installing new committedModel version="+committedModelClock);
+ }
+ committedModel = newCommittedModel;
+ committedModelClock = version;
+ }
+ }
+ }
+ numCommitting.decrementAndGet();
+ continue;
+ }
+
+
+ int id = rand.nextInt(ndocs);
+ Object sync = syncArr[id];
+
+ // set the lastId before we actually change it sometimes to try and
+ // uncover more race conditions between writing and reading
+ boolean before = rand.nextBoolean();
+ if (before) {
+ lastId = id;
+ }
+
+ // We can't concurrently update the same document and retain our invariants of increasing values
+ // since we can't guarantee what order the updates will be executed.
+ // Even with versions, we can't remove the sync because increasing versions does not mean increasing vals.
+ //
+ // NOTE: versioning means we can now remove the sync and tell what update "won"
+ // synchronized (sync) {
+ DocInfo info = model.get(id);
+
+ long val = info.val;
+ long nextVal = Math.abs(val)+1;
+
+ if (oper < commitPercent + deletePercent) {
+ verbose("deleting id",id,"val=",nextVal);
+
+ Long version = deleteAndGetVersion(Integer.toString(id), null);
+ assertTrue(version < 0);
+
+ // only update model if the version is newer
+ synchronized (model) {
+ DocInfo currInfo = model.get(id);
+ if (Math.abs(version) > Math.abs(currInfo.version)) {
+ model.put(id, new DocInfo(version, -nextVal));
+ }
+ }
+
+ verbose("deleting id", id, "val=",nextVal,"DONE");
+ } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) {
+ verbose("deleteByQyery id",id,"val=",nextVal);
+
+ Long version = deleteByQueryAndGetVersion("id:"+Integer.toString(id), null);
+ assertTrue(version < 0);
+
+ // only update model if the version is newer
+ synchronized (model) {
+ DocInfo currInfo = model.get(id);
+ if (Math.abs(version) > Math.abs(currInfo.version)) {
+ model.put(id, new DocInfo(version, -nextVal));
+ }
+ }
+
+ verbose("deleteByQyery id", id, "val=",nextVal,"DONE");
+ } else {
+ verbose("adding id", id, "val=", nextVal);
+
+ // assertU(adoc("id",Integer.toString(id), field, Long.toString(nextVal)));
+ Long version = addAndGetVersion(sdoc("id", Integer.toString(id), field, Long.toString(nextVal)), null);
+ assertTrue(version > 0);
+
+ // only update model if the version is newer
+ synchronized (model) {
+ DocInfo currInfo = model.get(id);
+ if (version > currInfo.version) {
+ model.put(id, new DocInfo(version, nextVal));
+ }
+ }
+
+ if (VERBOSE) {
+ verbose("adding id", id, "val=", nextVal,"DONE");
+ }
+
+ }
+ // } // end sync
+
+ if (!before) {
+ lastId = id;
+ }
+ }
+ } catch (Throwable e) {
+ operations.set(-1L);
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ threads.add(thread);
+ }
+
+
+ for (int i=0; i<nReadThreads; i++) {
+ Thread thread = new Thread("READER"+i) {
+ Random rand = new Random(random().nextInt());
+
+ @Override
+ public void run() {
+ try {
+ while (operations.decrementAndGet() >= 0) {
+ // bias toward a recently changed doc
+ int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs);
+
+ // when indexing, we update the index, then the model
+ // so when querying, we should first check the model, and then the index
+
+ boolean realTime = rand.nextInt(100) < percentRealtimeQuery;
+ DocInfo info;
+
+ if (realTime) {
+ info = model.get(id);
+ } else {
+ synchronized(globalLock) {
+ info = committedModel.get(id);
+ }
+ }
+
+ if (VERBOSE) {
+ verbose("querying id", id);
+ }
+ SolrQueryRequest sreq;
+ if (realTime) {
+ sreq = req("wt","json", "qt","/get", "ids",Integer.toString(id));
+ } else {
+ sreq = req("wt","json", "q","id:"+Integer.toString(id), "omitHeader","true");
+ }
+
+ String response = h.query(sreq);
+ Map rsp = (Map)ObjectBuilder.fromJSON(response);
+ List doclist = (List)(((Map)rsp.get("response")).get("docs"));
+ if (doclist.size() == 0) {
+ // there's no info we can get back with a delete, so not much we can check without further synchronization
+ } else {
+ assertEquals(1, doclist.size());
+ long foundVal = (Long)(((Map)doclist.get(0)).get(field));
+ long foundVer = (Long)(((Map)doclist.get(0)).get("_version_"));
+ if (foundVer < Math.abs(info.version)
+ || (foundVer == info.version && foundVal != info.val) ) { // if the version matches, the val must
+ verbose("ERROR, id=", id, "found=",response,"model",info);
+ assertTrue(false);
+ }
+ }
+ }
+ } catch (Throwable e) {
+ operations.set(-1L);
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ threads.add(thread);
+ }
+
+
+ for (Thread thread : threads) {
+ thread.start();
+ }
+
+ for (Thread thread : threads) {
+ thread.join();
+ }
+
+ }
+
+
+}