You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2015/03/06 15:37:44 UTC
svn commit: r1664633 - in /lucene/dev/trunk/lucene: CHANGES.txt
core/src/java/org/apache/lucene/index/CheckIndex.java
core/src/test/org/apache/lucene/index/TestCheckIndex.java
Author: rmuir
Date: Fri Mar 6 14:37:44 2015
New Revision: 1664633
URL: http://svn.apache.org/r1664633
Log:
LUCENE-6341: Add a -fast option to CheckIndex
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1664633&r1=1664632&r2=1664633&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Mar 6 14:37:44 2015
@@ -19,6 +19,8 @@ New Features
for counting ranges that align with the underlying terms as defined by the
NumberRangePrefixTree (e.g. familiar date units like days). (David Smiley)
+* LUCENE-6341: Add a -fast option to CheckIndex. (Robert Muir)
+
API Changes
* LUCENE-3312: The API of oal.document was restructured to
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1664633&r1=1664632&r2=1664633&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Fri Mar 6 14:37:44 2015
@@ -411,6 +411,20 @@ public class CheckIndex implements Close
}
private boolean verbose;
+
+ /** See {@link #getChecksumsOnly}. */
+ public boolean getChecksumsOnly() {
+ return checksumsOnly;
+ }
+
+ /**
+ * If true, only validate physical integrity for all files.
+ * Note that the returned nested status objects (e.g. storedFieldStatus) will be null. */
+ public void setChecksumsOnly(boolean v) {
+ checksumsOnly = v;
+ }
+
+ private boolean checksumsOnly;
/** Set infoStream where messages should go. If null, no
* messages are printed. If verbose is true then more
@@ -668,42 +682,45 @@ public class CheckIndex implements Close
}
}
- // Test Livedocs
- segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream, failFast);
-
- // Test Fieldinfos
- segInfoStat.fieldInfoStatus = testFieldInfos(reader, infoStream, failFast);
-
- // Test Field Norms
- segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream, failFast);
-
- // Test the Term Index
- segInfoStat.termIndexStatus = testPostings(reader, infoStream, verbose, failFast);
-
- // Test Stored Fields
- segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream, failFast);
-
- // Test Term Vectors
- segInfoStat.termVectorStatus = testTermVectors(reader, infoStream, verbose, crossCheckTermVectors, failFast);
-
- segInfoStat.docValuesStatus = testDocValues(reader, infoStream, failFast);
-
- // Rethrow the first exception we encountered
- // This will cause stats for failed segments to be incremented properly
- if (segInfoStat.liveDocStatus.error != null) {
- throw new RuntimeException("Live docs test failed");
- } else if (segInfoStat.fieldInfoStatus.error != null) {
- throw new RuntimeException("Field Info test failed");
- } else if (segInfoStat.fieldNormStatus.error != null) {
- throw new RuntimeException("Field Norm test failed");
- } else if (segInfoStat.termIndexStatus.error != null) {
- throw new RuntimeException("Term Index test failed");
- } else if (segInfoStat.storedFieldStatus.error != null) {
- throw new RuntimeException("Stored Field test failed");
- } else if (segInfoStat.termVectorStatus.error != null) {
- throw new RuntimeException("Term Vector test failed");
- } else if (segInfoStat.docValuesStatus.error != null) {
- throw new RuntimeException("DocValues test failed");
+ if (checksumsOnly == false) {
+ // Test Livedocs
+ segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream, failFast);
+
+ // Test Fieldinfos
+ segInfoStat.fieldInfoStatus = testFieldInfos(reader, infoStream, failFast);
+
+ // Test Field Norms
+ segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream, failFast);
+
+ // Test the Term Index
+ segInfoStat.termIndexStatus = testPostings(reader, infoStream, verbose, failFast);
+
+ // Test Stored Fields
+ segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream, failFast);
+
+ // Test Term Vectors
+ segInfoStat.termVectorStatus = testTermVectors(reader, infoStream, verbose, crossCheckTermVectors, failFast);
+
+ // Test Docvalues
+ segInfoStat.docValuesStatus = testDocValues(reader, infoStream, failFast);
+
+ // Rethrow the first exception we encountered
+ // This will cause stats for failed segments to be incremented properly
+ if (segInfoStat.liveDocStatus.error != null) {
+ throw new RuntimeException("Live docs test failed");
+ } else if (segInfoStat.fieldInfoStatus.error != null) {
+ throw new RuntimeException("Field Info test failed");
+ } else if (segInfoStat.fieldNormStatus.error != null) {
+ throw new RuntimeException("Field Norm test failed");
+ } else if (segInfoStat.termIndexStatus.error != null) {
+ throw new RuntimeException("Term Index test failed");
+ } else if (segInfoStat.storedFieldStatus.error != null) {
+ throw new RuntimeException("Stored Field test failed");
+ } else if (segInfoStat.termVectorStatus.error != null) {
+ throw new RuntimeException("Term Vector test failed");
+ } else if (segInfoStat.docValuesStatus.error != null) {
+ throw new RuntimeException("DocValues test failed");
+ }
}
msg(infoStream, "");
@@ -2110,13 +2127,16 @@ public class CheckIndex implements Close
boolean doExorcise = false;
boolean doCrossCheckTermVectors = false;
boolean verbose = false;
+ boolean doChecksumsOnly = false;
List<String> onlySegments = new ArrayList<>();
String indexPath = null;
String dirImpl = null;
int i = 0;
while(i < args.length) {
String arg = args[i];
- if ("-exorcise".equals(arg)) {
+ if ("-fast".equals(arg)) {
+ doChecksumsOnly = true;
+ } else if ("-exorcise".equals(arg)) {
doExorcise = true;
} else if ("-crossCheckTermVectors".equals(arg)) {
doCrossCheckTermVectors = true;
@@ -2151,6 +2171,7 @@ public class CheckIndex implements Close
System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-crossCheckTermVectors] [-segment X] [-segment Y] [-dir-impl X]\n" +
"\n" +
" -exorcise: actually write a new segments_N file, removing any problematic segments\n" +
+ " -fast: just verify file checksums, omitting logical integrity checks\n" +
" -crossCheckTermVectors: verifies that term vectors match postings; THIS IS VERY SLOW!\n" +
" -codec X: when exorcising, codec to write the new segments_N file with\n" +
" -verbose: print additional details\n" +
@@ -2185,6 +2206,11 @@ public class CheckIndex implements Close
System.out.println("ERROR: cannot specify both -exorcise and -segment");
return 1;
}
+
+ if (doChecksumsOnly && doCrossCheckTermVectors) {
+ System.out.println("ERROR: cannot specify both -fast and -crossCheckTermVectors");
+ return 1;
+ }
System.out.println("\nOpening index @ " + indexPath + "\n");
Directory directory = null;
@@ -2204,6 +2230,7 @@ public class CheckIndex implements Close
try (Directory dir = directory;
CheckIndex checker = new CheckIndex(dir)) {
checker.setCrossCheckTermVectors(doCrossCheckTermVectors);
+ checker.setChecksumsOnly(doChecksumsOnly);
checker.setInfoStream(System.out, verbose);
Status result = checker.checkIndex(onlySegments);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java?rev=1664633&r1=1664632&r2=1664633&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java Fri Mar 6 14:37:44 2015
@@ -24,9 +24,11 @@ import java.util.List;
import java.util.ArrayList;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.Token;
@@ -118,6 +120,54 @@ public class TestCheckIndex extends Luce
dir.close(); // checkindex
}
+ public void testChecksumsOnly() throws IOException {
+ LineFileDocs lf = new LineFileDocs(random());
+ Directory dir = newDirectory();
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(analyzer));
+ for (int i = 0; i < 100; i++) {
+ iw.addDocument(lf.nextDoc());
+ }
+ iw.addDocument(new Document());
+ iw.commit();
+ iw.close();
+ lf.close();
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
+ CheckIndex checker = new CheckIndex(dir);
+ checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8));
+ if (VERBOSE) checker.setInfoStream(System.out);
+ CheckIndex.Status indexStatus = checker.checkIndex();
+ assertTrue(indexStatus.clean);
+ checker.close();
+ dir.close();
+ analyzer.close();
+ }
+
+ public void testChecksumsOnlyVerbose() throws IOException {
+ LineFileDocs lf = new LineFileDocs(random());
+ Directory dir = newDirectory();
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(analyzer));
+ for (int i = 0; i < 100; i++) {
+ iw.addDocument(lf.nextDoc());
+ }
+ iw.addDocument(new Document());
+ iw.commit();
+ iw.close();
+ lf.close();
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
+ CheckIndex checker = new CheckIndex(dir);
+ checker.setInfoStream(new PrintStream(bos, true, IOUtils.UTF_8));
+ if (VERBOSE) checker.setInfoStream(System.out);
+ CheckIndex.Status indexStatus = checker.checkIndex();
+ assertTrue(indexStatus.clean);
+ checker.close();
+ dir.close();
+ analyzer.close();
+ }
+
public void testObtainsLock() throws IOException {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));