You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2012/01/03 17:11:13 UTC
svn commit: r1226846 - in /lucene/dev/branches/solrcloud/solr/core/src:
java/org/apache/solr/update/TransactionLog.java
test/org/apache/solr/search/TestRecovery.java
Author: yonik
Date: Tue Jan 3 16:11:13 2012
New Revision: 1226846
URL: http://svn.apache.org/viewvc?rev=1226846&view=rev
Log:
reverse tlog reading fixes + more tests
Modified:
lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/TransactionLog.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/search/TestRecovery.java
Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/TransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/TransactionLog.java?rev=1226846&r1=1226845&r2=1226846&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/TransactionLog.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/TransactionLog.java Tue Jan 3 16:11:13 2012
@@ -558,7 +558,7 @@ public class TransactionLog {
fis = new ChannelFastInputStream(channel, 0);
if (sz >=4) {
- readHeader(fis);
+ // readHeader(fis); // should not be needed
prevPos = sz - 4;
fis.seek(prevPos);
nextLength = fis.readInt();
@@ -589,7 +589,7 @@ public class TransactionLog {
} else {
// Position buffer so that this record is at the end.
// For small records, this will cause subsequent calls to next() to be within the buffer.
- long seekPos = endOfThisRecord - 8192;
+ long seekPos = endOfThisRecord - fis.getBufferSize();
seekPos = Math.min(seekPos, prevPos); // seek to the start of the record if it's larger then the block size.
seekPos = Math.max(seekPos, 0);
fis.seek(seekPos);
@@ -602,6 +602,9 @@ public class TransactionLog {
// TODO: optionally skip document data
Object o = codec.readVal(fis);
+ if (fis.position() != prevPos + 4 + thisLength) {
+ System.out.println("set breakpoint here");
+ }
assert fis.position() == prevPos + 4 + thisLength; // this is only true if we read all the data
return o;
@@ -632,22 +635,18 @@ public class TransactionLog {
class ChannelFastInputStream extends FastInputStream {
private FileChannel ch;
- private long chPosition;
public ChannelFastInputStream(FileChannel ch, long chPosition) {
+ // super(null, new byte[10],0,0); // a small buffer size for testing purposes // nocommit!!! this currently causes failures!
super(null);
this.ch = ch;
- this.chPosition = chPosition;
- super.readFromStream = chPosition; // make sure position() method returns the correct value
+ super.readFromStream = chPosition;
}
@Override
public int readWrappedStream(byte[] target, int offset, int len) throws IOException {
ByteBuffer bb = ByteBuffer.wrap(target, offset, len);
- int ret = ch.read(bb, chPosition);
- if (ret >= 0) {
- chPosition += ret;
- }
+ int ret = ch.read(bb, readFromStream);
return ret;
}
@@ -657,7 +656,6 @@ class ChannelFastInputStream extends Fas
pos = (int)(position - readFromStream);
} else {
readFromStream = position;
- chPosition = position;
end = pos = 0;
}
}
@@ -667,6 +665,10 @@ class ChannelFastInputStream extends Fas
return readFromStream - end;
}
+ public int getBufferSize() {
+ return buf.length;
+ }
+
@Override
public void close() throws IOException {
ch.close();
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/search/TestRecovery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/search/TestRecovery.java?rev=1226846&r1=1226845&r2=1226846&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/search/TestRecovery.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/search/TestRecovery.java Tue Jan 3 16:11:13 2012
@@ -29,8 +29,7 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
@@ -75,10 +74,13 @@ public class TestRecovery extends SolrTe
clearIndex();
assertU(commit());
- assertU(adoc("id","1"));
- assertJQ(req("q","id:1")
- ,"/response/numFound==0"
- );
+ Deque<Long> versions = new ArrayDeque<Long>();
+ versions.addFirst( addAndGetVersion(sdoc("id","1") , null) );
+ versions.addFirst( addAndGetVersion(sdoc("id", "11"), null));
+
+ assertJQ(req("q","*:*"),"/response/numFound==0");
+
+ assertJQ(req("qt","/get", "getVersions",""+versions.size()) ,"/versions==" + versions);
h.close();
createCore();
@@ -87,24 +89,31 @@ public class TestRecovery extends SolrTe
// verify that previous close didn't do a commit
// recovery should be blocked by our hook
- assertJQ(req("q","id:1") ,"/response/numFound==0");
+ assertJQ(req("q","*:*") ,"/response/numFound==0");
+
+ // make sure we can still access versions after a restart
+ assertJQ(req("qt","/get", "getVersions",""+versions.size()),"/versions==" + versions);
// unblock recovery
logReplay.release(1000);
+ // make sure we can still access versions during recovery
+ assertJQ(req("qt","/get", "getVersions",""+versions.size()),"/versions==" + versions);
+
// wait until recovery has finished
assertTrue(logReplayFinish.tryAcquire(60, TimeUnit.SECONDS));
- assertJQ(req("q", "id:1")
- , "/response/numFound==1"
- );
+ assertJQ(req("q","*:*") ,"/response/numFound==2");
+
+ // make sure we can still access versions after recovery
+ assertJQ(req("qt","/get", "getVersions",""+versions.size()) ,"/versions==" + versions);
assertU(adoc("id","2"));
assertU(adoc("id","3"));
assertU(delI("2"));
assertU(adoc("id","4"));
- assertJQ(req("q","*:*") ,"/response/numFound==1");
+ assertJQ(req("q","*:*") ,"/response/numFound==2");
h.close();
createCore();
@@ -113,7 +122,7 @@ public class TestRecovery extends SolrTe
// wait until recovery has finished
assertTrue(logReplayFinish.tryAcquire(60, TimeUnit.SECONDS));
- assertJQ(req("q","*:*") ,"/response/numFound==3");
+ assertJQ(req("q","*:*") ,"/response/numFound==4");
assertJQ(req("q","id:2") ,"/response/numFound==0");
// no updates, so insure that recovery does not run
@@ -123,7 +132,7 @@ public class TestRecovery extends SolrTe
// Solr should kick this off now
// h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();
- assertJQ(req("q","*:*") ,"/response/numFound==3");
+ assertJQ(req("q","*:*") ,"/response/numFound==4");
Thread.sleep(100);
assertEquals(permits, logReplay.availablePermits()); // no updates, so insure that recovery didn't run