You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/01/30 22:19:57 UTC
svn commit: r1238024 [2/2] - in /lucene/dev/branches/lucene2858: ./ lucene/
lucene/contrib/ lucene/contrib/misc/src/java/
lucene/src/java/org/apache/lucene/codecs/
lucene/src/java/org/apache/lucene/codecs/appending/
lucene/src/java/org/apache/lucene/co...
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java Mon Jan 30 21:19:54 2012
@@ -31,7 +31,6 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.Directory;
/** This postings format "inlines" the postings for terms that have
* low docFreq. It wraps another postings format, which is used for
@@ -115,8 +114,8 @@ public abstract class PulsingPostingsFor
}
@Override
- public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
- wrappedPostingsBaseFormat.files(dir, segmentInfo, segmentSuffix, files);
- BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files);
+ public void files(SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
+ wrappedPostingsBaseFormat.files(segmentInfo, segmentSuffix, files);
+ BlockTreeTermsReader.files(segmentInfo, segmentSuffix, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/sep/SepDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/sep/SepDocValuesConsumer.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/sep/SepDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/sep/SepDocValuesConsumer.java Mon Jan 30 21:19:54 2012
@@ -38,6 +38,7 @@ import org.apache.lucene.util.IOUtils;
public class SepDocValuesConsumer extends DocValuesWriterBase {
private final Directory directory;
private final FieldInfos fieldInfos;
+
public SepDocValuesConsumer(PerDocWriteState state) throws IOException {
super(state);
this.directory = state.directory;
@@ -49,9 +50,9 @@ public class SepDocValuesConsumer extend
return directory;
}
- public static void files(Directory dir, SegmentInfo segmentInfo,
+ public static void files(SegmentInfo segmentInfo,
Set<String> files) throws IOException {
- files(dir, segmentInfo.getFieldInfos(), segmentInfo.name, files);
+ files(segmentInfo.dir, segmentInfo.getFieldInfos(), segmentInfo.name, files);
}
@SuppressWarnings("fallthrough")
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java Mon Jan 30 21:19:54 2012
@@ -24,7 +24,6 @@ import org.apache.lucene.codecs.FieldInf
import org.apache.lucene.codecs.FieldInfosReader;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.Directory;
/**
* plaintext field infos format
@@ -47,7 +46,7 @@ public class SimpleTextFieldInfosFormat
}
@Override
- public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
- SimpleTextFieldInfosReader.files(dir, info, files);
+ public void files(SegmentInfo info, Set<String> files) throws IOException {
+ SimpleTextFieldInfosReader.files(info, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java Mon Jan 30 21:19:54 2012
@@ -133,7 +133,7 @@ public class SimpleTextFieldInfosReader
return new String(scratch.bytes, scratch.offset+offset, scratch.length-offset, IOUtils.CHARSET_UTF_8);
}
- public static void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
+ public static void files(SegmentInfo info, Set<String> files) throws IOException {
files.add(IndexFileNames.segmentFileName(info.name, "", FIELD_INFOS_EXTENSION));
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java Mon Jan 30 21:19:54 2012
@@ -138,7 +138,7 @@ public class SimpleTextLiveDocsFormat ex
}
@Override
- public void separateFiles(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
+ public void files(SegmentInfo info, Set<String> files) throws IOException {
if (info.hasDeletions()) {
files.add(IndexFileNames.fileNameFromGeneration(info.name, LIVEDOCS_EXTENSION, info.getDelGen()));
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsConsumer.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsConsumer.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsConsumer.java Mon Jan 30 21:19:54 2012
@@ -280,7 +280,7 @@ public class SimpleTextNormsConsumer ext
}
}
- public static void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
+ public static void files(SegmentInfo info, Set<String> files) throws IOException {
FieldInfos fieldInfos = info.getFieldInfos();
for (FieldInfo fieldInfo : fieldInfos) {
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java Mon Jan 30 21:19:54 2012
@@ -26,7 +26,6 @@ import org.apache.lucene.codecs.PerDocPr
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.store.Directory;
/**
* plain-text norms format
@@ -47,9 +46,7 @@ public class SimpleTextNormsFormat exten
}
@Override
- public void files(Directory dir, SegmentInfo info, Set<String> files)
- throws IOException {
- SimpleTextNormsConsumer.files(dir, info, files);
-
+ public void files(SegmentInfo info, Set<String> files) throws IOException {
+ SimpleTextNormsConsumer.files(info, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPostingsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPostingsFormat.java Mon Jan 30 21:19:54 2012
@@ -27,7 +27,6 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.store.Directory;
/** For debugging, curiosity, transparency only!! Do not
* use this codec in production.
@@ -61,7 +60,7 @@ public class SimpleTextPostingsFormat ex
}
@Override
- public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
+ public void files(SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
files.add(getPostingsFileName(segmentInfo.name, segmentSuffix));
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsFormat.java Mon Jan 30 21:19:54 2012
@@ -47,7 +47,7 @@ public class SimpleTextStoredFieldsForma
}
@Override
- public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
- SimpleTextStoredFieldsReader.files(dir, info, files);
+ public void files(SegmentInfo info, Set<String> files) throws IOException {
+ SimpleTextStoredFieldsReader.files(info, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java Mon Jan 30 21:19:54 2012
@@ -178,7 +178,7 @@ public class SimpleTextStoredFieldsReade
}
}
- public static void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
+ public static void files(SegmentInfo info, Set<String> files) throws IOException {
files.add(IndexFileNames.segmentFileName(info.name, "", SimpleTextStoredFieldsWriter.FIELDS_EXTENSION));
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsFormat.java Mon Jan 30 21:19:54 2012
@@ -47,7 +47,7 @@ public class SimpleTextTermVectorsFormat
}
@Override
- public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
- SimpleTextTermVectorsReader.files(dir, info, files);
+ public void files(SegmentInfo info, Set<String> files) throws IOException {
+ SimpleTextTermVectorsReader.files(info, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java Mon Jan 30 21:19:54 2012
@@ -201,7 +201,7 @@ public class SimpleTextTermVectorsReader
}
}
- public static void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
+ public static void files(SegmentInfo info, Set<String> files) throws IOException {
if (info.getHasVectors()) {
files.add(IndexFileNames.segmentFileName(info.name, "", VECTORS_EXTENSION));
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Mon Jan 30 21:19:54 2012
@@ -494,14 +494,11 @@ public class DocumentsWriterPerThread {
}
if (infoStream.isEnabled("DWPT")) {
- final double newSegmentSizeNoStore = newSegment.sizeInBytes(false)/1024./1024.;
- final double newSegmentSize = newSegment.sizeInBytes(true)/1024./1024.;
+ final double newSegmentSize = newSegment.sizeInBytes()/1024./1024.;
infoStream.message("DWPT", "flushed: segment=" + newSegment +
" ramUsed=" + nf.format(startMBUsed) + " MB" +
- " newFlushedSize=" + nf.format(newSegmentSize) + " MB" +
- " (" + nf.format(newSegmentSizeNoStore) + " MB w/o doc stores)" +
- " docs/MB=" + nf.format(flushedDocCount / newSegmentSize) +
- " new/old=" + nf.format(100.0 * newSegmentSizeNoStore / startMBUsed) + "%");
+ " newFlushedSize(includes docstores)=" + nf.format(newSegmentSize) + " MB" +
+ " docs/MB=" + nf.format(flushedDocCount / newSegmentSize));
}
doAfterFlush();
success = true;
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/IndexFileNames.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/IndexFileNames.java Mon Jan 30 21:19:54 2012
@@ -54,9 +54,6 @@ public final class IndexFileNames {
/** Extension of compound file entries */
public static final String COMPOUND_FILE_ENTRIES_EXTENSION = "cfe";
- /** Extension of compound file for doc store files*/
- public static final String COMPOUND_FILE_STORE_EXTENSION = "cfx";
-
/**
* This array contains all filename extensions used by
* Lucene's index files, with one exception, namely the
@@ -68,7 +65,6 @@ public final class IndexFileNames {
COMPOUND_FILE_EXTENSION,
COMPOUND_FILE_ENTRIES_EXTENSION,
GEN_EXTENSION,
- COMPOUND_FILE_STORE_EXTENSION,
};
/**
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon Jan 30 21:19:54 2012
@@ -2545,27 +2545,27 @@ public class IndexWriter implements Clos
// only relevant for segments that share doc store with others,
// because the DS might have been copied already, in which case we
// just want to update the DS name of this SegmentInfo.
- // NOTE: pre-3x segments include a null DSName if they don't share doc
- // store. The following code ensures we don't accidentally insert
- // 'null' to the map.
String dsName = info.getDocStoreSegment();
+ assert dsName != null;
final String newDsName;
- if (dsName != null) {
- if (dsNames.containsKey(dsName)) {
- newDsName = dsNames.get(dsName);
- } else {
- dsNames.put(dsName, segName);
- newDsName = segName;
- }
+ if (dsNames.containsKey(dsName)) {
+ newDsName = dsNames.get(dsName);
} else {
+ dsNames.put(dsName, segName);
newDsName = segName;
}
- Set<String> codecDocStoreFiles = info.codecDocStoreFiles();
+ Set<String> codecDocStoreFiles = new HashSet<String>();
+ if (info.getDocStoreOffset() != -1) {
+ // only violate the codec this way if its preflex
+ codec.storedFieldsFormat().files(info, codecDocStoreFiles);
+ codec.termVectorsFormat().files(info, codecDocStoreFiles);
+ }
+
// Copy the segment files
for (String file: info.files()) {
final String newFileName;
- if (codecDocStoreFiles.contains(file) || file.endsWith(IndexFileNames.COMPOUND_FILE_STORE_EXTENSION)) {
+ if (codecDocStoreFiles.contains(file)) {
newFileName = newDsName + IndexFileNames.stripSegmentName(file);
if (dsFilesCopied.contains(newFileName)) {
continue;
@@ -4070,12 +4070,11 @@ public class IndexWriter implements Clos
*/
static final Collection<String> createCompoundFile(Directory directory, String fileName, CheckAbort checkAbort, final SegmentInfo info, IOContext context)
throws IOException {
-
+ assert info.getDocStoreOffset() == -1;
// Now merge all added files
Collection<String> files = info.files();
CompoundFileDirectory cfsDir = new CompoundFileDirectory(directory, fileName, context, true);
try {
- assert assertNoSeparateFiles(files, directory, info);
for (String file : files) {
directory.copy(cfsDir, file, file, context);
checkAbort.work(directory.fileLength(file));
@@ -4086,20 +4085,4 @@ public class IndexWriter implements Clos
return files;
}
-
-
- /**
- * used only by assert: checks that filenames about to be put in cfs belong.
- */
- private static boolean assertNoSeparateFiles(Collection<String> files,
- Directory dir, SegmentInfo info) throws IOException {
- // maybe this is overkill, but codec naming clashes would be bad.
- Set<String> separateFiles = new HashSet<String>();
- info.getCodec().separateFiles(dir, info, separateFiles);
-
- for (String file : files) {
- assert !separateFiles.contains(file) : file + " should not go in CFS!";
- }
- return true;
- }
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Mon Jan 30 21:19:54 2012
@@ -48,16 +48,12 @@ final class SegmentCoreReaders {
// SegmentReaders:
private final AtomicInteger ref = new AtomicInteger(1);
- final String segment;
final FieldInfos fieldInfos;
final FieldsProducer fields;
final PerDocProducer perDocProducer;
final PerDocProducer norms;
- final Directory dir;
- final Directory cfsDir;
- final IOContext context;
final int termsIndexDivisor;
private final SegmentReader owner;
@@ -65,7 +61,6 @@ final class SegmentCoreReaders {
final StoredFieldsReader fieldsReaderOrig;
final TermVectorsReader termVectorsReaderOrig;
final CompoundFileDirectory cfsReader;
- final CompoundFileDirectory storeCFSReader;
final CloseableThreadLocal<StoredFieldsReader> fieldsReaderLocal = new CloseableThreadLocal<StoredFieldsReader>() {
@Override
@@ -91,22 +86,18 @@ final class SegmentCoreReaders {
throw new IllegalArgumentException("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)");
}
- segment = si.name;
final Codec codec = si.getCodec();
- this.context = context;
- this.dir = dir;
-
+ final Directory cfsDir; // confusing name: if (cfs) its the cfsdir, otherwise its the segment's directory.
+
boolean success = false;
try {
- Directory dir0 = dir;
if (si.getUseCompoundFile()) {
- cfsReader = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
- dir0 = cfsReader;
+ cfsDir = cfsReader = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
} else {
cfsReader = null;
+ cfsDir = dir;
}
- cfsDir = dir0;
si.loadFieldInfos(cfsDir, false); // prevent opening the CFS to load fieldInfos
fieldInfos = si.getFieldInfos();
@@ -119,36 +110,13 @@ final class SegmentCoreReaders {
// ask codec for its Norms:
// TODO: since we don't write any norms file if there are no norms,
// kinda jaky to assume the codec handles the case of no norms file at all gracefully?!
- norms = codec.normsFormat().docsProducer(segmentReadState, dir);
+ norms = codec.normsFormat().docsProducer(segmentReadState);
perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState);
-
- final Directory storeDir;
- if (si.getDocStoreOffset() != -1) {
- if (si.getDocStoreIsCompoundFile()) {
- storeCFSReader = new CompoundFileDirectory(dir,
- IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION),
- context, false);
- storeDir = storeCFSReader;
- assert storeDir != null;
- } else {
- storeCFSReader = null;
- storeDir = dir;
- assert storeDir != null;
- }
- } else if (si.getUseCompoundFile()) {
- storeDir = cfsReader;
- storeCFSReader = null;
- assert storeDir != null;
- } else {
- storeDir = dir;
- storeCFSReader = null;
- assert storeDir != null;
- }
-
- fieldsReaderOrig = si.getCodec().storedFieldsFormat().fieldsReader(storeDir, si, fieldInfos, context);
+
+ fieldsReaderOrig = si.getCodec().storedFieldsFormat().fieldsReader(cfsDir, si, fieldInfos, context);
if (si.getHasVectors()) { // open term vector files only as needed
- termVectorsReaderOrig = si.getCodec().termVectorsFormat().vectorsReader(storeDir, si, fieldInfos, context);
+ termVectorsReaderOrig = si.getCodec().termVectorsFormat().vectorsReader(cfsDir, si, fieldInfos, context);
} else {
termVectorsReaderOrig = null;
}
@@ -175,7 +143,7 @@ final class SegmentCoreReaders {
//System.out.println("core.decRef seg=" + owner.getSegmentInfo() + " rc=" + ref);
if (ref.decrementAndGet() == 0) {
IOUtils.close(termVectorsLocal, fieldsReaderLocal, fields, perDocProducer,
- termVectorsReaderOrig, fieldsReaderOrig, cfsReader, storeCFSReader, norms);
+ termVectorsReaderOrig, fieldsReaderOrig, cfsReader, norms);
notifyCoreClosedListeners();
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Mon Jan 30 21:19:54 2012
@@ -33,7 +33,6 @@ import org.apache.lucene.store.CompoundF
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.StringHelper;
/**
* Information about a segment such as it's name, directory, and files related
@@ -74,8 +73,7 @@ public final class SegmentInfo implement
private volatile List<String> files; // cached list of files that this segment uses
// in the Directory
- private volatile long sizeInBytesNoStore = -1; // total byte size of all but the store files (computed on demand)
- private volatile long sizeInBytesWithStore = -1; // total byte size of all of our files (computed on demand)
+ private volatile long sizeInBytes = -1; // total byte size of all files (computed on demand)
//TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
private int docStoreOffset; // if this segment shares stored fields & vectors, this
@@ -213,63 +211,12 @@ public final class SegmentInfo implement
* Returns total size in bytes of all of files used by this segment
*/
public long sizeInBytes() throws IOException {
- return sizeInBytes(true);
- }
-
- /**
- * Returns total size in bytes of all of files used by this segment (if
- * {@code includeDocStores} is true), or the size of all files except the
- * store files otherwise.
- * <p>
- * NOTE: includeDocStores=false should only be used for debugging.
- * Theoretically a codec could combine its files however it wants (after-
- * the-fact or something), and this calculation is not particularly
- * efficient.
- */
- long sizeInBytes(boolean includeDocStores) throws IOException {
- // TODO: based on how this is used, can't we just forget about all this docstore crap?
- // its really an abstraction violation into the codec
- if (includeDocStores) {
- if (sizeInBytesWithStore != -1) {
- return sizeInBytesWithStore;
- }
long sum = 0;
for (final String fileName : files()) {
- // We don't count bytes used by a shared doc store
- // against this segment
- if (docStoreOffset == -1 || !isDocStoreFile(fileName)) {
- sum += dir.fileLength(fileName);
- }
- }
- sizeInBytesWithStore = sum;
- return sizeInBytesWithStore;
- } else {
- if (sizeInBytesNoStore != -1) {
- return sizeInBytesNoStore;
- }
- long sum = 0;
- for (final String fileName : files()) {
- if (isDocStoreFile(fileName)) {
- continue;
- }
sum += dir.fileLength(fileName);
}
- sizeInBytesNoStore = sum;
- return sizeInBytesNoStore;
- }
- }
-
- Set<String> codecDocStoreFiles() throws IOException {
- Set<String> docStoreFiles = new HashSet<String>();
- codec.storedFieldsFormat().files(dir, this, docStoreFiles);
- codec.termVectorsFormat().files(dir, this, docStoreFiles);
- return docStoreFiles;
- }
-
- // TODO: a little messy, but sizeInBytes above that uses this is the real problem.
- private boolean isDocStoreFile(String fileName) throws IOException {
- Set<String> docStoreFiles = codecDocStoreFiles();
- return fileName.endsWith(IndexFileNames.COMPOUND_FILE_STORE_EXTENSION) || docStoreFiles.contains(fileName);
+ sizeInBytes = sum;
+ return sizeInBytes;
}
public boolean getHasVectors() throws IOException {
@@ -473,30 +420,7 @@ public final class SegmentInfo implement
}
final Set<String> fileSet = new HashSet<String>();
- boolean useCompoundFile = getUseCompoundFile();
-
- if (useCompoundFile) {
- fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.COMPOUND_FILE_EXTENSION));
- if (version != null && StringHelper.getVersionComparator().compare("4.0", version) <= 0) {
- fileSet.add(IndexFileNames.segmentFileName(name, "",
- IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
- }
- } else {
- codec.files(dir, this, fileSet);
- }
-
- // regardless of compound file setting: these files are always in the directory
- codec.separateFiles(dir, this, fileSet);
-
- if (docStoreOffset != -1) {
- // We are sharing doc stores (stored fields, term
- // vectors) with other segments
- assert docStoreSegment != null;
- // TODO: push this out into preflex fieldsFormat?
- if (docStoreIsCompoundFile) {
- fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION));
- }
- }
+ codec.files(this, fileSet);
files = new ArrayList<String>(fileSet);
@@ -507,8 +431,7 @@ public final class SegmentInfo implement
* files this segment has. */
private void clearFilesCache() {
files = null;
- sizeInBytesNoStore = -1;
- sizeInBytesWithStore = -1;
+ sizeInBytes = -1;
}
/** {@inheritDoc} */
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/index/SegmentReader.java Mon Jan 30 21:19:54 2012
@@ -181,14 +181,14 @@ public final class SegmentReader extends
public String toString() {
// SegmentInfo.toString takes dir and number of
// *pending* deletions; so we reverse compute that here:
- return si.toString(core.dir, si.docCount - numDocs - si.getDelCount());
+ return si.toString(si.dir, si.docCount - numDocs - si.getDelCount());
}
/**
* Return the name of the segment this reader is reading.
*/
public String getSegmentName() {
- return core.segment;
+ return si.name;
}
/**
@@ -203,7 +203,7 @@ public final class SegmentReader extends
// Don't ensureOpen here -- in certain cases, when a
// cloned/reopened reader needs to commit, it may call
// this method on the closed original reader
- return core.dir;
+ return si.dir;
}
// This is necessary so that cloned SegmentReaders (which
Modified: lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/util/fst/FST.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/java/org/apache/lucene/util/fst/FST.java Mon Jan 30 21:19:54 2012
@@ -231,7 +231,7 @@ public final class FST<T> {
b.append(" hasOutput");
}
if (flag(BIT_ARC_HAS_FINAL_OUTPUT)) {
- b.append(" hasOutput");
+ b.append(" hasFinalOutput");
}
if (bytesPerArc != 0) {
b.append(" arcArray(idx=" + arcIdx + " of " + numArcs + ")");
@@ -1447,6 +1447,7 @@ public final class FST<T> {
// Find top nodes with highest number of incoming arcs:
NodeQueue q = new NodeQueue(topN);
+ // TODO: we could use more RAM efficient selection algo here...
NodeAndInCount bottom = null;
for(int node=0;node<inCounts.length;node++) {
if (inCounts[node] >= minInCountDeref) {
@@ -1515,6 +1516,8 @@ public final class FST<T> {
int addressError = 0;
+ //int totWasted = 0;
+
// Since we re-reverse the bytes, we now write the
// nodes backwards, so that BIT_TARGET_NEXT is
// unchanged:
@@ -1554,10 +1557,11 @@ public final class FST<T> {
writer.writeByte(ARCS_AS_FIXED_ARRAY);
writer.writeVInt(arc.numArcs);
writer.writeVInt(bytesPerArc);
+ //System.out.println("node " + node + ": " + arc.numArcs + " arcs");
}
int maxBytesPerArc = 0;
-
+ //int wasted = 0;
while(true) { // iterate over all arcs for this node
//System.out.println(" arc label=" + arc.label + " target=" + arc.target + " pos=" + writer.posWrite);
@@ -1680,6 +1684,7 @@ public final class FST<T> {
// incoming FST did... but in this case we
// will retry (below) so it's OK to ovewrite
// bytes:
+ //wasted += bytesPerArc - arcBytes;
writer.setPosWrite(arcStartPos + bytesPerArc);
}
@@ -1693,6 +1698,8 @@ public final class FST<T> {
if (useArcArray) {
if (maxBytesPerArc == bytesPerArc || (retry && maxBytesPerArc <= bytesPerArc)) {
// converged
+ //System.out.println(" bba=" + bytesPerArc + " wasted=" + wasted);
+ //totWasted += wasted;
break;
}
} else {
@@ -1719,6 +1726,7 @@ public final class FST<T> {
// other nodes because we only produce acyclic FSTs
// w/ nodes only pointing "forwards":
assert !negDelta;
+ //System.out.println("TOT wasted=" + totWasted);
// Converged!
break;
}
@@ -1730,7 +1738,7 @@ public final class FST<T> {
}
fst.startNode = newNodeAddress[startNode];
- //System.out.println("new startNode=" + startNode);
+ //System.out.println("new startNode=" + fst.startNode + " old startNode=" + startNode);
if (emptyOutput != null) {
fst.setEmptyOutput(emptyOutput);
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/lucene40ords/Lucene40WithOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/lucene40ords/Lucene40WithOrds.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/lucene40ords/Lucene40WithOrds.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/lucene40ords/Lucene40WithOrds.java Mon Jan 30 21:19:54 2012
@@ -36,7 +36,6 @@ import org.apache.lucene.codecs.lucene40
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
// TODO: we could make separate base class that can wrapp
@@ -137,9 +136,9 @@ public class Lucene40WithOrds extends Po
static final String PROX_EXTENSION = "prx";
@Override
- public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
- Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files);
- BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files);
- FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files);
+ public void files(SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
+ Lucene40PostingsReader.files(segmentInfo, segmentSuffix, files);
+ BlockTermsReader.files(segmentInfo, segmentSuffix, files);
+ FixedGapTermsIndexReader.files(segmentInfo, segmentSuffix, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java Mon Jan 30 21:19:54 2012
@@ -202,9 +202,9 @@ public class MockFixedIntBlockPostingsFo
}
@Override
- public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
+ public void files(SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
SepPostingsReader.files(segmentInfo, segmentSuffix, files);
- BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files);
- FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files);
+ BlockTermsReader.files(segmentInfo, segmentSuffix, files);
+ FixedGapTermsIndexReader.files(segmentInfo, segmentSuffix, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java Mon Jan 30 21:19:54 2012
@@ -225,9 +225,9 @@ public class MockVariableIntBlockPosting
}
@Override
- public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
+ public void files(SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
SepPostingsReader.files(segmentInfo, segmentSuffix, files);
- BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files);
- FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files);
+ BlockTermsReader.files(segmentInfo, segmentSuffix, files);
+ FixedGapTermsIndexReader.files(segmentInfo, segmentSuffix, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Mon Jan 30 21:19:54 2012
@@ -413,20 +413,20 @@ public class MockRandomPostingsFormat ex
}
@Override
- public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
+ public void files(SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
final String seedFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SEED_EXT);
files.add(seedFileName);
SepPostingsReader.files(segmentInfo, segmentSuffix, files);
- Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files);
- BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files);
- BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files);
- FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files);
- VariableGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files);
+ Lucene40PostingsReader.files(segmentInfo, segmentSuffix, files);
+ BlockTermsReader.files(segmentInfo, segmentSuffix, files);
+ BlockTreeTermsReader.files(segmentInfo, segmentSuffix, files);
+ FixedGapTermsIndexReader.files(segmentInfo, segmentSuffix, files);
+ VariableGapTermsIndexReader.files(segmentInfo, segmentSuffix, files);
// hackish!
Iterator<String> it = files.iterator();
while(it.hasNext()) {
final String file = it.next();
- if (!dir.fileExists(file)) {
+ if (!segmentInfo.dir.fileExists(file)) {
it.remove();
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mocksep/MockSepDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mocksep/MockSepDocValuesFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mocksep/MockSepDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mocksep/MockSepDocValuesFormat.java Mon Jan 30 21:19:54 2012
@@ -28,7 +28,6 @@ import org.apache.lucene.codecs.sep.SepD
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.store.Directory;
/**
* Separate-file docvalues implementation
@@ -48,7 +47,7 @@ public class MockSepDocValuesFormat exte
}
@Override
- public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
- SepDocValuesConsumer.files(dir, info, files);
+ public void files(SegmentInfo info, Set<String> files) throws IOException {
+ SepDocValuesConsumer.files(info, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java Mon Jan 30 21:19:54 2012
@@ -37,7 +37,6 @@ import org.apache.lucene.codecs.sep.SepP
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
/**
@@ -130,9 +129,9 @@ public class MockSepPostingsFormat exten
}
@Override
- public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
+ public void files(SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
SepPostingsReader.files(segmentInfo, segmentSuffix, files);
- BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files);
- FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files);
+ BlockTermsReader.files(segmentInfo, segmentSuffix, files);
+ FixedGapTermsIndexReader.files(segmentInfo, segmentSuffix, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java Mon Jan 30 21:19:54 2012
@@ -34,7 +34,6 @@ import org.apache.lucene.codecs.pulsing.
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.Directory;
/**
* Pulsing(1, Pulsing(2, Lucene40))
@@ -92,8 +91,8 @@ public class NestedPulsingPostingsFormat
}
@Override
- public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
- Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files);
- BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files);
+ public void files(SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) throws IOException {
+ Lucene40PostingsReader.files(segmentInfo, segmentSuffix, files);
+ BlockTreeTermsReader.files(segmentInfo, segmentSuffix, files);
}
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java Mon Jan 30 21:19:54 2012
@@ -17,17 +17,23 @@ package org.apache.lucene.codecs.preflex
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Set;
+
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
-import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfosFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.lucene3x.Lucene3xCodec;
+import org.apache.lucene.codecs.lucene3x.Lucene3xNormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.StringHelper;
/**
* Writes 3.x-like indexes (not perfect emulation yet) for testing only!
@@ -35,14 +41,13 @@ import org.apache.lucene.util.LuceneTest
*/
public class PreFlexRWCodec extends Lucene3xCodec {
private final PostingsFormat postings = new PreFlexRWPostingsFormat();
- private final NormsFormat norms = new PreFlexRWNormsFormat();
+ private final Lucene3xNormsFormat norms = new PreFlexRWNormsFormat();
private final FieldInfosFormat fieldInfos = new PreFlexRWFieldInfosFormat();
private final TermVectorsFormat termVectors = new PreFlexRWTermVectorsFormat();
private final SegmentInfosFormat segmentInfos = new PreFlexRWSegmentInfosFormat();
+ private final StoredFieldsFormat storedFields = new PreFlexRWStoredFieldsFormat();
// TODO: this should really be a different impl
private final LiveDocsFormat liveDocs = new Lucene40LiveDocsFormat();
- // TODO: this should really be a different impl
- private final StoredFieldsFormat storedFields = new Lucene40StoredFieldsFormat();
@Override
public PostingsFormat postingsFormat() {
@@ -54,7 +59,7 @@ public class PreFlexRWCodec extends Luce
}
@Override
- public NormsFormat normsFormat() {
+ public Lucene3xNormsFormat normsFormat() {
if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) {
return norms;
} else {
@@ -106,4 +111,19 @@ public class PreFlexRWCodec extends Luce
return super.storedFieldsFormat();
}
}
+
+ @Override
+ public void files(SegmentInfo info, Set<String> files) throws IOException {
+ if (info.getUseCompoundFile() && LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) {
+ // because we don't fully emulate 3.x codec, PreFlexRW actually writes 4.x format CFS files.
+ // so we must check segment version here to see if its a "real" 3.x segment or a "fake"
+ // one that we wrote with a 4.x-format CFS+CFE, in this case we must add the .CFE
+ String version = info.getVersion();
+ if (version != null && StringHelper.getVersionComparator().compare("4.0", version) <= 0) {
+ files.add(IndexFileNames.segmentFileName(info.name, "", IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ }
+ }
+
+ super.files(info, files);
+ }
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsWriter.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsWriter.java Mon Jan 30 21:19:54 2012
@@ -202,7 +202,7 @@ public final class PreFlexRWTermVectorsW
// throw an exception to prevent the corruption from
// entering the index. See LUCENE-1282 for
// details.
- throw new RuntimeException("mergeVectors produced an invalid result: mergedDocs is " + numDocs + " but tvx size is " + tvx.getFilePointer() + " file=" + tvx.toString() + "; now aborting this merge to prevent index corruption");
+ throw new RuntimeException("tvx size mismatch: mergedDocs is " + numDocs + " but tvx size is " + tvx.getFilePointer() + " file=" + tvx.toString() + "; now aborting this merge to prevent index corruption");
}
/** Close all streams. */
Modified: lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test-framework/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java Mon Jan 30 21:19:54 2012
@@ -46,7 +46,6 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Bits;
@@ -592,7 +591,7 @@ public class RAMOnlyPostingsFormat exten
}
@Override
- public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) {
+ public void files(SegmentInfo segmentInfo, String segmentSuffix, Set<String> files) {
final String idFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, ID_EXTENSION);
files.add(idFileName);
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Mon Jan 30 21:19:54 2012
@@ -1548,7 +1548,7 @@ public class TestIndexWriter extends Luc
SegmentInfo s = ((SegmentReader) r).getSegmentInfo();
assertFalse(s.getHasVectors());
Set<String> files = new HashSet<String>();
- s.getCodec().termVectorsFormat().files(dir, s, files);
+ s.getCodec().termVectorsFormat().files(s, files);
assertTrue(files.isEmpty());
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Mon Jan 30 21:19:54 2012
@@ -135,34 +135,4 @@ public class TestSegmentMerger extends L
TestSegmentReader.checkNorms(mergedReader);
mergedReader.close();
}
-
- // LUCENE-3143
- public void testInvalidFilesToCreateCompound() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
- IndexWriter w = new IndexWriter(dir, iwc);
-
- // Create an index w/ .del file
- w.addDocument(new Document());
- Document doc = new Document();
- doc.add(new TextField("c", "test"));
- w.addDocument(doc);
- w.commit();
- w.deleteDocuments(new Term("c", "test"));
- w.close();
-
- // Assert that SM fails if .del exists
- SegmentMerger sm = new SegmentMerger(InfoStream.getDefault(), dir, 1, "a", MergeState.CheckAbort.NONE, null, null, Codec.getDefault(), newIOContext(random));
- boolean doFail = false;
- try {
- IndexWriter.createCompoundFile(dir, "b1", MergeState.CheckAbort.NONE, w.segmentInfos.info(0), newIOContext(random));
- doFail = true; // should never get here
- } catch (AssertionError e) {
- // expected
- }
- assertFalse("should not have been able to create a .cfs with .del and .s* files", doFail);
-
- dir.close();
- }
-
}
Modified: lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java Mon Jan 30 21:19:54 2012
@@ -189,7 +189,7 @@ public class TestTermVectorsReader exten
SegmentInfo s = ((SegmentReader) r).getSegmentInfo();
assertTrue(s.getHasVectors());
Set<String> files = new HashSet<String>();
- s.getCodec().termVectorsFormat().files(dir, s, files);
+ s.getCodec().termVectorsFormat().files(s, files);
assertFalse(files.isEmpty());
for (String file : files) {
assertTrue(dir.fileExists(file));
Modified: lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/branches/lucene2858/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java Mon Jan 30 21:19:54 2012
@@ -530,7 +530,7 @@ public class TestFSTs extends LuceneTest
if (VERBOSE) {
System.out.println("TEST: now rewrite");
}
- final FST<T> packed =fst.pack(_TestUtil.nextInt(random, 1, 10), _TestUtil.nextInt(random, 0, 10000000));
+ final FST<T> packed = fst.pack(_TestUtil.nextInt(random, 1, 10), _TestUtil.nextInt(random, 0, 10000000));
if (VERBOSE) {
System.out.println("TEST: now verify packed FST");
}
@@ -1308,13 +1308,13 @@ public class TestFSTs extends LuceneTest
System.out.println("Pack...");
fst = fst.pack(4, 100000000);
System.out.println("New size " + fst.sizeInBytes() + " bytes");
- } else {
- Directory dir = FSDirectory.open(new File(dirOut));
- IndexOutput out = dir.createOutput("fst.bin", IOContext.DEFAULT);
- fst.save(out);
- out.close();
- System.out.println("Saved FST to fst.bin.");
}
+
+ Directory dir = FSDirectory.open(new File(dirOut));
+ IndexOutput out = dir.createOutput("fst.bin", IOContext.DEFAULT);
+ fst.save(out);
+ out.close();
+ System.out.println("Saved FST to fst.bin.");
if (!verify) {
return;
Modified: lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene2858/solr/core/src/java/org/apache/solr/cloud/Overseer.java Mon Jan 30 21:19:54 2012
@@ -176,20 +176,24 @@ public class Overseer implements NodeSta
return false;
}
/**
- * Try to assign core to the cluster
+ * Try to assign core to the cluster.
* @throws KeeperException
* @throws InterruptedException
*/
private CloudState updateState(CloudState state, String nodeName, CoreState coreState) throws KeeperException, InterruptedException {
String collection = coreState.getCollectionName();
String zkCoreNodeName = coreState.getCoreNodeName();
-
- String shardId;
- if (coreState.getProperties().get(ZkStateReader.SHARD_ID_PROP) == null) {
- shardId = AssignShard.assignShard(collection, state);
- } else {
- shardId = coreState.getProperties().get(ZkStateReader.SHARD_ID_PROP);
- }
+
+ // use the provided non null shardId
+ String shardId = coreState.getProperties().get(ZkStateReader.SHARD_ID_PROP);
+ if(shardId==null) {
+ //use shardId from CloudState
+ shardId = getAssignedId(state, nodeName, coreState);
+ }
+ if(shardId==null) {
+ //request new shardId
+ shardId = AssignShard.assignShard(collection, state);
+ }
Map<String,String> props = new HashMap<String,String>();
for (Entry<String,String> entry : coreState.getProperties().entrySet()) {
@@ -209,6 +213,23 @@ public class Overseer implements NodeSta
CloudState newCloudState = updateSlice(state, collection, slice);
return newCloudState;
}
+
+ /*
+ * Return an already assigned id or null if not assigned
+ */
+ private String getAssignedId(final CloudState state, final String nodeName,
+ final CoreState coreState) {
+ final String key = coreState.getProperties().get(ZkStateReader.NODE_NAME_PROP) + "_" + coreState.getProperties().get(ZkStateReader.CORE_NAME_PROP);
+ Map<String, Slice> slices = state.getSlices(coreState.getCollectionName());
+ if (slices != null) {
+ for (Slice slice : slices.values()) {
+ if (slice.getShards().get(key) != null) {
+ return slice.getName();
+ }
+ }
+ }
+ return null;
+ }
private CloudState updateSlice(CloudState state, String collection, Slice slice) {
@@ -480,6 +501,7 @@ public class Overseer implements NodeSta
Set<String> downNodes = complement(oldLiveNodes, liveNodes);
for(String node: downNodes) {
NodeStateWatcher watcher = nodeStateWatches.remove(node);
+ log.debug("Removed NodeStateWatcher for node:" + node);
}
}
@@ -491,6 +513,7 @@ public class Overseer implements NodeSta
if (!nodeStateWatches.containsKey(nodeName)) {
zkCmdExecutor.ensureExists(path, zkClient);
nodeStateWatches.put(nodeName, new NodeStateWatcher(zkClient, nodeName, path, this));
+ log.debug("Added NodeStateWatcher for node " + nodeName);
} else {
log.debug("watch already added");
}
Modified: lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1238024&r1=1238023&r2=1238024&view=diff
==============================================================================
--- lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/branches/lucene2858/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Mon Jan 30 21:19:54 2012
@@ -40,6 +40,7 @@ import org.apache.solr.core.CoreDescript
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.Code;
+import org.apache.zookeeper.data.Stat;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -49,6 +50,54 @@ public class OverseerTest extends SolrTe
private static final boolean DEBUG = false;
+ private static class MockZKController{
+
+ private final SolrZkClient zkClient;
+ private final String nodeName;
+
+ public MockZKController(String zkAddress, String nodeName) throws InterruptedException, TimeoutException, IOException, KeeperException {
+ this.nodeName = nodeName;
+ zkClient = new SolrZkClient(zkAddress, TIMEOUT);
+ Overseer.createClientNodes(zkClient, nodeName);
+
+ // live node
+ final String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + "node1";
+ zkClient.makePath(nodePath, CreateMode.EPHEMERAL, true);
+ }
+
+ private void deleteNode(final String path) {
+ try {
+ Stat stat = zkClient.exists(path, null, false);
+ zkClient.delete(path, stat.getVersion(), false);
+ } catch (KeeperException e) {
+ fail("Unexpected KeeperException!" + e);
+ } catch (InterruptedException e) {
+ fail("Unexpected InterruptedException!" + e);
+ }
+ }
+
+ public void close(){
+ try {
+ deleteNode(ZkStateReader.LIVE_NODES_ZKNODE + "/" + "node1");
+ zkClient.close();
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ public void publishState(String coreName, String stateName) throws KeeperException, InterruptedException{
+ HashMap<String,String> coreProps = new HashMap<String,String>();
+ coreProps.put(ZkStateReader.STATE_PROP, stateName);
+ coreProps.put(ZkStateReader.NODE_NAME_PROP, nodeName);
+ coreProps.put(ZkStateReader.CORE_NAME_PROP, coreName);
+ CoreState state = new CoreState(coreName, "collection1", coreProps);
+ final String statePath = Overseer.STATES_NODE + "/" + nodeName;
+ zkClient.setData(statePath, ZkStateReader.toJSON(new CoreState[] {state}), true);
+ }
+
+ }
+
@BeforeClass
public static void beforeClass() throws Exception {
initCore();
@@ -438,11 +487,11 @@ public class OverseerTest extends SolrTe
SolrZkClient controllerClient = null;
SolrZkClient overseerClient = null;
ZkStateReader reader = null;
+ MockZKController mockController = null;
try {
server.run();
controllerClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
-
AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
controllerClient.makePath(ZkStateReader.LIVE_NODES_ZKNODE, true);
@@ -450,45 +499,35 @@ public class OverseerTest extends SolrTe
reader = new ZkStateReader(controllerClient);
reader.createClusterStateWatchersAndUpdate();
- Overseer.createClientNodes(controllerClient, "node1");
- overseerClient = electNewOverseer(server.getZkAddress());
-
- // live node
- final String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + "node1";
- controllerClient.makePath(nodePath, CreateMode.EPHEMERAL, true);
+ mockController = new MockZKController(server.getZkAddress(), "node1");
- HashMap<String,String> coreProps = new HashMap<String,String>();
- coreProps.put(ZkStateReader.STATE_PROP, ZkStateReader.RECOVERING);
- coreProps.put(ZkStateReader.NODE_NAME_PROP, "node1");
- CoreState state = new CoreState("core1", "collection1", coreProps);
-
- final String statePath = Overseer.STATES_NODE + "/node1";
- // publish node state (recovering)
- controllerClient.setData(statePath, ZkStateReader.toJSON(new CoreState[] {state}), true);
+ overseerClient = electNewOverseer(server.getZkAddress());
+
+ mockController.publishState("core1", ZkStateReader.RECOVERING);
// wait overseer assignment
waitForSliceCount(reader, "collection1", 1);
verifyStatus(reader, ZkStateReader.RECOVERING);
- // publish node state (active)
- coreProps.put(ZkStateReader.STATE_PROP, ZkStateReader.ACTIVE);
- coreProps.put(ZkStateReader.SHARD_ID_PROP, "shard1");
- state = new CoreState("core1", "collection1", coreProps);
- controllerClient.setData(statePath,
- ZkStateReader.toJSON(new CoreState[] {state}), true);
+ int version = getCloudStateVersion(controllerClient);
+
+ mockController.publishState("core1", ZkStateReader.ACTIVE);
+
+ while(version == getCloudStateVersion(controllerClient));
verifyStatus(reader, ZkStateReader.ACTIVE);
+ version = getCloudStateVersion(controllerClient);
overseerClient.close();
-
- coreProps.put(ZkStateReader.STATE_PROP, ZkStateReader.RECOVERING);
- state = new CoreState("core1", "collection1", coreProps);
-
- controllerClient.setData(statePath,
- ZkStateReader.toJSON(new CoreState[] {state}), true);
+ Thread.sleep(1000); //wait for overseer to get killed
- overseerClient = electNewOverseer(server.getZkAddress());
+ mockController.publishState("core1", ZkStateReader.RECOVERING);
+ version = getCloudStateVersion(controllerClient);
+ overseerClient = electNewOverseer(server.getZkAddress());
+
+ while(version == getCloudStateVersion(controllerClient));
+
verifyStatus(reader, ZkStateReader.RECOVERING);
assertEquals("Live nodes count does not match", 1, reader.getCloudState()
@@ -497,6 +536,10 @@ public class OverseerTest extends SolrTe
.getSlice("collection1", "shard1").getShards().size());
} finally {
+ if (mockController != null) {
+ mockController.close();
+ }
+
if (overseerClient != null) {
overseerClient.close();
}
@@ -509,6 +552,80 @@ public class OverseerTest extends SolrTe
server.shutdown();
}
}
+
+ @Test
+ public void testDoubleAssignment() throws Exception {
+ String zkDir = dataDir.getAbsolutePath() + File.separator
+ + "zookeeper/server1/data";
+
+ System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "2");
+
+ ZkTestServer server = new ZkTestServer(zkDir);
+
+ SolrZkClient controllerClient = null;
+ SolrZkClient overseerClient = null;
+ ZkStateReader reader = null;
+ MockZKController mockController = null;
+
+ try {
+ server.run();
+ controllerClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
+
+ AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
+ AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
+ controllerClient.makePath(ZkStateReader.LIVE_NODES_ZKNODE, true);
+
+ reader = new ZkStateReader(controllerClient);
+ reader.createClusterStateWatchersAndUpdate();
+
+ mockController = new MockZKController(server.getZkAddress(), "node1");
+
+ overseerClient = electNewOverseer(server.getZkAddress());
+
+ mockController.publishState("core1", ZkStateReader.RECOVERING);
+
+ // wait overseer assignment
+ waitForSliceCount(reader, "collection1", 1);
+
+ verifyStatus(reader, ZkStateReader.RECOVERING);
+
+ mockController.close();
+
+ int version = getCloudStateVersion(controllerClient);
+
+ mockController = new MockZKController(server.getZkAddress(), "node1");
+ mockController.publishState("core1", ZkStateReader.RECOVERING);
+
+ while (version == getCloudStateVersion(controllerClient));
+
+ reader.updateCloudState(true);
+ CloudState state = reader.getCloudState();
+ assertEquals("more than 1 shard id was assigned to same core", 1, state.getSlices("collection1").size());
+
+ } finally {
+ System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
+ if (overseerClient != null) {
+ overseerClient.close();
+ }
+ if (mockController != null) {
+ mockController.close();
+ }
+
+ if (controllerClient != null) {
+ controllerClient.close();
+ }
+ if (reader != null) {
+ reader.close();
+ }
+ server.shutdown();
+ }
+ }
+
+ private int getCloudStateVersion(SolrZkClient controllerClient)
+ throws KeeperException, InterruptedException {
+ return controllerClient.exists(ZkStateReader.CLUSTER_STATE, null, false).getVersion();
+ }
+
private SolrZkClient electNewOverseer(String address) throws InterruptedException,
TimeoutException, IOException, KeeperException {