You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2013/07/14 17:55:34 UTC
svn commit: r1502987 - in /lucene/sandbox/luke: ./ lib/ lib/solr/
src/org/apache/lucene/index/ src/org/apache/lucene/luke/core/
src/org/apache/lucene/luke/core/decoders/ src/org/apache/lucene/luke/ui/
Author: markrmiller
Date: Sun Jul 14 15:55:32 2013
New Revision: 1502987
URL: http://svn.apache.org/r1502987
Log:
LUCENE-2562: hack and slash update to Lucene/Solr 4.3.1
Added:
lucene/sandbox/luke/lib/lucene-analyzers-common-4.3.1.jar (with props)
lucene/sandbox/luke/lib/lucene-codecs-4.3.1.jar (with props)
lucene/sandbox/luke/lib/lucene-core-4.3.1.jar (with props)
lucene/sandbox/luke/lib/lucene-misc-4.3.1.jar (with props)
lucene/sandbox/luke/lib/lucene-queries-4.3.1.jar (with props)
lucene/sandbox/luke/lib/lucene-queryparser-4.3.1.jar (with props)
lucene/sandbox/luke/lib/solr/solr-core-4.3.1.jar (with props)
lucene/sandbox/luke/lib/solr/solr-solrj-4.3.1.jar (with props)
Removed:
lucene/sandbox/luke/lib/lucene-analyzers-common-4.0-dev.jar
lucene/sandbox/luke/lib/lucene-core-4.0-dev.jar
lucene/sandbox/luke/lib/lucene-misc-4.0-dev.jar
lucene/sandbox/luke/lib/lucene-queries-4.0-dev.jar
lucene/sandbox/luke/lib/lucene-xml-query-parser-4.0-dev.jar
lucene/sandbox/luke/lib/solr/apache-solr-core-4.0-dev.jar
lucene/sandbox/luke/lib/solr/apache-solr-solrj-4.0-dev.jar
lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/OldDateFieldDecoder.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/OldNumberToolsDecoder.java
Modified:
lucene/sandbox/luke/.classpath
lucene/sandbox/luke/src/org/apache/lucene/index/IndexGate.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/AccessibleTopHitCollector.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/AllHitsCollector.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/HighFreqTerms.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/IndexInfo.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/KeepAllIndexDeletionPolicy.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/KeepLastIndexDeletionPolicy.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/NoScoringScorer.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/TermStats.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/Util.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/BinaryDecoder.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/DateDecoder.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/Decoder.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/NumIntDecoder.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/NumLongDecoder.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/SolrDecoder.java
lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/StringDecoder.java
lucene/sandbox/luke/src/org/apache/lucene/luke/ui/AnalyzersTab.java
lucene/sandbox/luke/src/org/apache/lucene/luke/ui/DocumentsTab.java
lucene/sandbox/luke/src/org/apache/lucene/luke/ui/FilesTab.java
lucene/sandbox/luke/src/org/apache/lucene/luke/ui/LukeWindow.java
lucene/sandbox/luke/src/org/apache/lucene/luke/ui/OverviewTab.java
lucene/sandbox/luke/src/org/apache/lucene/luke/ui/SearchTab.java
Modified: lucene/sandbox/luke/.classpath
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/.classpath?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/.classpath (original)
+++ lucene/sandbox/luke/.classpath Sun Jul 14 15:55:32 2013
@@ -28,12 +28,13 @@
<classpathentry kind="lib" path="lib/pivot-web-server-2.0.jar"/>
<classpathentry kind="lib" path="lib/pivot-wtk-2.0.jar" sourcepath="/Pivot"/>
<classpathentry kind="lib" path="lib/pivot-wtk-terra-2.0.jar"/>
- <classpathentry kind="lib" path="lib/lucene-analyzers-common-4.0-dev.jar"/>
- <classpathentry kind="lib" path="lib/lucene-core-4.0-dev.jar" sourcepath="/solcene-cloud-lwe"/>
- <classpathentry kind="lib" path="lib/lucene-misc-4.0-dev.jar"/>
- <classpathentry kind="lib" path="lib/lucene-queries-4.0-dev.jar"/>
- <classpathentry kind="lib" path="lib/lucene-xml-query-parser-4.0-dev.jar"/>
- <classpathentry kind="lib" path="lib/solr/apache-solr-core-4.0-dev.jar"/>
- <classpathentry kind="lib" path="lib/solr/apache-solr-solrj-4.0-dev.jar"/>
+ <classpathentry kind="lib" path="lib/lucene-analyzers-common-4.3.1.jar"/>
+ <classpathentry kind="lib" path="lib/lucene-core-4.3.1.jar" sourcepath="/lucene-solr-5x-4"/>
+ <classpathentry kind="lib" path="lib/lucene-misc-4.3.1.jar"/>
+ <classpathentry kind="lib" path="lib/lucene-queries-4.3.1.jar"/>
+ <classpathentry kind="lib" path="lib/lucene-queryparser-4.3.1.jar"/>
+ <classpathentry kind="lib" path="lib/solr/solr-core-4.3.1.jar"/>
+ <classpathentry kind="lib" path="lib/solr/solr-solrj-4.3.1.jar"/>
+ <classpathentry kind="lib" path="lib/lucene-codecs-4.3.1.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
Added: lucene/sandbox/luke/lib/lucene-analyzers-common-4.3.1.jar
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/lib/lucene-analyzers-common-4.3.1.jar?rev=1502987&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/sandbox/luke/lib/lucene-analyzers-common-4.3.1.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/sandbox/luke/lib/lucene-codecs-4.3.1.jar
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/lib/lucene-codecs-4.3.1.jar?rev=1502987&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/sandbox/luke/lib/lucene-codecs-4.3.1.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/sandbox/luke/lib/lucene-core-4.3.1.jar
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/lib/lucene-core-4.3.1.jar?rev=1502987&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/sandbox/luke/lib/lucene-core-4.3.1.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/sandbox/luke/lib/lucene-misc-4.3.1.jar
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/lib/lucene-misc-4.3.1.jar?rev=1502987&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/sandbox/luke/lib/lucene-misc-4.3.1.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/sandbox/luke/lib/lucene-queries-4.3.1.jar
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/lib/lucene-queries-4.3.1.jar?rev=1502987&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/sandbox/luke/lib/lucene-queries-4.3.1.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/sandbox/luke/lib/lucene-queryparser-4.3.1.jar
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/lib/lucene-queryparser-4.3.1.jar?rev=1502987&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/sandbox/luke/lib/lucene-queryparser-4.3.1.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/sandbox/luke/lib/solr/solr-core-4.3.1.jar
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/lib/solr/solr-core-4.3.1.jar?rev=1502987&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/sandbox/luke/lib/solr/solr-core-4.3.1.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/sandbox/luke/lib/solr/solr-solrj-4.3.1.jar
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/lib/solr/solr-solrj-4.3.1.jar?rev=1502987&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/sandbox/luke/lib/solr/solr-solrj-4.3.1.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: lucene/sandbox/luke/src/org/apache/lucene/index/IndexGate.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/index/IndexGate.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/index/IndexGate.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/index/IndexGate.java Sun Jul 14 15:55:32 2013
@@ -18,19 +18,18 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.io.PrintStream;
import java.lang.reflect.Field;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Set;
-import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
-import org.apache.lucene.index.codecs.standard.StandardCodec;
-import org.apache.lucene.luke.core.KeepAllIndexDeletionPolicy;
+import org.apache.lucene.codecs.Codec;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
/**
@@ -40,40 +39,38 @@ import org.apache.lucene.store.IndexInpu
*/
public class IndexGate {
static Field deletable = null;
- static Field hasChanges = null;
- static PrintStream infoStream = IndexWriter.getDefaultInfoStream();
+
+ //static PrintStream infoStream = IndexWriter.getDefaultInfoStream();
static HashMap<String, String> knownExtensions = new HashMap<String, String>();
static {
- knownExtensions.put(IndexFileNames.COMPOUND_FILE_EXTENSION, "compound file with various index data");
- knownExtensions.put(IndexFileNames.COMPOUND_FILE_STORE_EXTENSION, "compound shared doc store file");
- knownExtensions.put(IndexFileNames.DELETES_EXTENSION, "list of deleted documents");
- knownExtensions.put(IndexFileNames.FIELD_INFOS_EXTENSION, "field names / infos");
- knownExtensions.put(IndexFileNames.FIELDS_EXTENSION, "stored fields data");
- knownExtensions.put(IndexFileNames.FIELDS_INDEX_EXTENSION, "stored fields index data");
- knownExtensions.put(IndexFileNames.GEN_EXTENSION, "generation number - global file");
- knownExtensions.put(IndexFileNames.NORMS_EXTENSION, "norms data for all fields");
- knownExtensions.put(IndexFileNames.SEGMENTS, "per-commit list of segments");
- knownExtensions.put(IndexFileNames.SEPARATE_NORMS_EXTENSION, "separate per-field norms data");
- knownExtensions.put(IndexFileNames.VECTORS_DOCUMENTS_EXTENSION, "term vectors document data");
- knownExtensions.put(IndexFileNames.VECTORS_FIELDS_EXTENSION, "term vector field data");
- knownExtensions.put(IndexFileNames.VECTORS_INDEX_EXTENSION, "term vectors index");
- CodecProvider codecs = CodecProvider.getDefault();
- HashSet<String> std = new HashSet<String>();
- StandardCodec.getStandardExtensions(std);
- for (String ext : codecs.getAllExtensions()) {
- if (knownExtensions.containsKey(ext)) {
- continue;
- } else {
- knownExtensions.put(ext, "codec-specific data" + (std.contains(ext) ? " (StandardCodec)" : ""));
- }
- }
+// knownExtensions.put(IndexFileNames.COMPOUND_FILE_EXTENSION, "compound file with various index data");
+// knownExtensions.put(IndexFileNames.COMPOUND_FILE_STORE_EXTENSION, "compound shared doc store file");
+// knownExtensions.put(IndexFileNames.DELETES_EXTENSION, "list of deleted documents");
+// knownExtensions.put(IndexFileNames.FIELD_INFOS_EXTENSION, "field names / infos");
+// knownExtensions.put(IndexFileNames.FIELDS_EXTENSION, "stored fields data");
+// knownExtensions.put(IndexFileNames.FIELDS_INDEX_EXTENSION, "stored fields index data");
+// knownExtensions.put(IndexFileNames.GEN_EXTENSION, "generation number - global file");
+// knownExtensions.put(IndexFileNames.NORMS_EXTENSION, "norms data for all fields");
+// knownExtensions.put(IndexFileNames.SEGMENTS, "per-commit list of segments");
+// knownExtensions.put(IndexFileNames.SEPARATE_NORMS_EXTENSION, "separate per-field norms data");
+// knownExtensions.put(IndexFileNames.VECTORS_DOCUMENTS_EXTENSION, "term vectors document data");
+// knownExtensions.put(IndexFileNames.VECTORS_FIELDS_EXTENSION, "term vector field data");
+// knownExtensions.put(IndexFileNames.VECTORS_INDEX_EXTENSION, "term vectors index");
+// CodecF codecs = CodecProvider.getDefault();
+// HashSet<String> std = new HashSet<String>();
+// StandardCodec.getStandardExtensions(std);
+// for (String ext : codecs.getAllExtensions()) {
+// if (knownExtensions.containsKey(ext)) {
+// continue;
+// } else {
+// knownExtensions.put(ext, "codec-specific data" + (std.contains(ext) ? " (StandardCodec)" : ""));
+// }
+// }
try {
deletable = IndexFileDeleter.class.getDeclaredField("deletable");
deletable.setAccessible(true);
- hasChanges = IndexReader.class.getDeclaredField("hasChanges");
- hasChanges.setAccessible(true);
} catch (Exception e) {
e.printStackTrace();
}
@@ -110,7 +107,7 @@ public class IndexGate {
protected Object doBody(String segmentsFile) throws CorruptIndexException,
IOException {
- IndexInput in = dir.openInput(segmentsFile);
+ IndexInput in = dir.openInput(segmentsFile, IOContext.READONCE);
Integer indexFormat = new Integer(in.readInt());
in.close();
return indexFormat;
@@ -136,10 +133,11 @@ public class IndexGate {
res.genericName = "Lucene 3.x or prior";
break;
}
- if (DefaultSegmentInfosWriter.FORMAT_CURRENT > format) {
- res.capabilities = null;
- res.genericName = "UNKNOWN";
- }
+ // TODO: this gate cannot reach into Lucene42FieldInfosFormat, it's in a codec sub package
+// if (Lucene42FieldInfosFormat.FORMAT_CURRENT > format) {
+// res.capabilities = null;
+// res.genericName = "UNKNOWN";
+// }
return res;
}
@@ -148,7 +146,7 @@ public class IndexGate {
infos.read(dir);
int compound = 0, nonCompound = 0;
for (int i = 0; i < infos.size(); i++) {
- if (((SegmentInfo)infos.get(i)).getUseCompoundFile()) {
+ if (((SegmentInfoPerCommit)infos.info(i)).info.getUseCompoundFile()) {
compound++;
} else {
nonCompound++;
@@ -157,31 +155,29 @@ public class IndexGate {
return compound > nonCompound;
}
- public static void deletePendingFiles(Directory dir, IndexDeletionPolicy policy) throws Exception {
- SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
- IndexFileDeleter deleter = new IndexFileDeleter(dir, policy, infos, infoStream, null, CodecProvider.getDefault());
- deleter.close();
- }
-
public static List<String> getDeletableFiles(Directory dir) throws Exception {
- SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
- IndexFileDeleter deleter = new IndexFileDeleter(dir, new KeepAllIndexDeletionPolicy(), infos, infoStream, null, CodecProvider.getDefault());
- return (List<String>)deletable.get(deleter);
+ List<String> known = getIndexFiles(dir);
+ Set<String> dirFiles = new HashSet<String>(Arrays.asList(dir.listAll()));
+ dirFiles.removeAll(known);
+ return new ArrayList<String>(dirFiles);
}
public static List<String> getIndexFiles(Directory dir) throws Exception {
- SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
- ArrayList<String> names = new ArrayList<String>();
- for (int i = 0; i < infos.size(); i++) {
- SegmentInfo info = (SegmentInfo)infos.info(i);
- names.addAll(info.files());
- names.add(info.getDelFileName());
+ List<IndexCommit> commits = null;
+ try {
+ commits = DirectoryReader.listCommits(dir);
+ } catch (IndexNotFoundException e) {
+ return Collections.emptyList();
+ }
+ Set<String> known = new HashSet<String>();
+ for (IndexCommit ic : commits) {
+ known.addAll(ic.getFileNames());
+ }
+ if (dir.fileExists(IndexFileNames.SEGMENTS_GEN)) {
+ known.add(IndexFileNames.SEGMENTS_GEN);
}
- names.add(infos.getCurrentSegmentFileName());
- names.add(IndexFileNames.SEGMENTS_GEN);
+ List<String> names = new ArrayList<String>(known);
+ Collections.sort(names);
return names;
}
@@ -191,7 +187,7 @@ public class IndexGate {
SegmentInfos infos = new SegmentInfos();
infos.read(dir);
- SegmentInfo info = (SegmentInfo) infos.info(0);
+ SegmentInfo info = (SegmentInfo) infos.info(0).info;
return info.getCodec();
}
@@ -200,16 +196,5 @@ public class IndexGate {
public String genericName = "N/A";
public String capabilities = "N/A";
}
-
- public static boolean hasChanges(IndexReader ir) {
- if (ir == null) {
- return false;
- }
- try {
- return hasChanges.getBoolean(ir);
- } catch (Exception e) {
- return false;
- }
- }
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/AccessibleTopHitCollector.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/AccessibleTopHitCollector.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/AccessibleTopHitCollector.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/AccessibleTopHitCollector.java Sun Jul 14 15:55:32 2013
@@ -19,7 +19,7 @@ package org.apache.lucene.luke.core;
import java.io.IOException;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
@@ -68,9 +68,9 @@ public class AccessibleTopHitCollector e
}
@Override
- public void setNextReader(IndexReader r, int base) throws IOException {
- this.docBase = base;
- tdc.setNextReader(r, base);
+ public void setNextReader(AtomicReaderContext context) throws IOException {
+ this.docBase = context.docBase;
+ tdc.setNextReader(context);
}
@Override
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/AllHitsCollector.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/AllHitsCollector.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/AllHitsCollector.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/AllHitsCollector.java Sun Jul 14 15:55:32 2013
@@ -21,7 +21,7 @@ package org.apache.lucene.luke.core;
import java.io.IOException;
import java.util.ArrayList;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Scorer;
class AllHitsCollector extends AccessibleHitCollector {
@@ -73,8 +73,8 @@ class AllHitsCollector extends Accessibl
}
@Override
- public void setNextReader(IndexReader r, int docBase) throws IOException {
- this.docBase = docBase;
+ public void setNextReader(AtomicReaderContext context) throws IOException {
+ this.docBase = context.docBase;
}
@Override
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/HighFreqTerms.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/HighFreqTerms.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/HighFreqTerms.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/HighFreqTerms.java Sun Jul 14 15:55:32 2013
@@ -17,21 +17,20 @@ package org.apache.lucene.luke.core;
* limitations under the License.
*/
+import java.io.File;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Iterator;
+
+import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiFields;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.FieldReaderException;
-import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.PriorityQueue;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Bits;
-import java.io.File;
-import java.util.Arrays;
-import java.util.Comparator;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PriorityQueue;
/**
* <b>NOTE: this is a temporary copy of contrib/misc from Lucene - we can get
@@ -75,7 +74,7 @@ public class HighFreqTerms {
}
}
String[] fields = field != null ? new String[] {field} : null;
- reader = IndexReader.open(dir, true);
+ reader = IndexReader.open(dir);
TermStats[] terms = getHighFreqTerms(reader, numTerms, fields);
if (!IncludeTermFreqs) {
// default HighFreqTerms behavior
@@ -111,34 +110,34 @@ public class HighFreqTerms {
public static TermStats[] getHighFreqTerms(IndexReader reader, int numTerms,
String[] fieldNames) throws Exception {
TermStatsQueue tiq = null;
+ TermsEnum te = null;
if (fieldNames != null) {
Fields fields = MultiFields.getFields(reader);
if (fields == null) {
- throw new FieldReaderException("no fields found for this index");
+ throw new RuntimeException("no fields found for this index");
}
tiq = new TermStatsQueue(numTerms);
for (String field : fieldNames) {
Terms terms = fields.terms(field);
if (terms != null) {
- TermsEnum termsEnum = terms.iterator();
- fillQueue(termsEnum, tiq, field);
+ te = terms.iterator(te);
+ fillQueue(te, tiq, field);
}
}
} else {
Fields fields = MultiFields.getFields(reader);
if (fields == null) {
- throw new FieldReaderException("no fields found for this index");
+ throw new RuntimeException("no fields found for this index");
}
tiq = new TermStatsQueue(numTerms);
- FieldsEnum fieldsEnum = fields.iterator();
+ Iterator<String> fieldsEnum = fields.iterator();
while (true) {
String field = fieldsEnum.next();
- if (field != null) {
- TermsEnum terms = fieldsEnum.terms();
- fillQueue(terms, tiq, field);
- } else {
- break;
+ Terms terms = fields.terms(field);
+ if (terms != null) {
+ te = terms.iterator(te);
+ fillQueue(te, tiq, field);
}
}
}
@@ -182,27 +181,17 @@ public class HighFreqTerms {
return ts;
}
- public static long getTotalTermFreq(IndexReader reader, String field,
- BytesRef termtext) throws Exception {
- BytesRef br = termtext;
+ public static long getTotalTermFreq(IndexReader reader, String field, BytesRef termtext) {
+ //BytesRef br = termtext;
long totalTF = 0;
- Bits skipDocs = MultiFields.getDeletedDocs(reader);
- DocsEnum de = MultiFields.getTermDocsEnum(reader, skipDocs, field, br);
- // if term is not in index return totalTF of 0
- if (de == null) {
+ try {
+ Bits liveDocs = MultiFields.getLiveDocs(reader);
+ totalTF = reader.getSumTotalTermFreq(field);
+ return totalTF;
+ } catch (Exception e) {
+ e.printStackTrace();
return 0;
}
- // use DocsEnum.read() and BulkResult api
- final DocsEnum.BulkReadResult bulkresult = de.getBulkResult();
- int count;
- while ((count = de.read()) != 0) {
- final int[] freqs = bulkresult.freqs.ints;
- final int limit = bulkresult.freqs.offset + count;
- for (int i = bulkresult.freqs.offset; i < limit; i++) {
- totalTF += freqs[i];
- }
- }
- return totalTF;
}
public static void fillQueue(TermsEnum termsEnum, TermStatsQueue tiq,
@@ -245,7 +234,7 @@ final class TotalTermFreqComparatorSortD
**/
final class TermStatsQueue extends PriorityQueue<TermStats> {
TermStatsQueue(int size) {
- initialize(size);
+ super(size);
}
@Override
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/IndexInfo.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/IndexInfo.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/IndexInfo.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/IndexInfo.java Sun Jul 14 15:55:32 2013
@@ -19,19 +19,24 @@ package org.apache.lucene.luke.core;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Date;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.CompositeReader;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexGate;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
@@ -50,47 +55,65 @@ public class IndexInfo {
private Codec indexCodec;
private boolean readOnly;
private IndexSearcher indexSearcher;
+ private int indexFormat;
- public IndexInfo(IndexReader reader, String indexPath, boolean readOnly) throws Exception {
+ public IndexInfo(IndexReader reader, String indexPath) throws Exception {
this.reader = reader;
this.indexSearcher = new IndexSearcher(reader);
- this.readOnly = readOnly;
- try {
- this.dir = reader.directory();
+ this.dir = null;
+ this.dirImpl = "N/A";
+ if (reader instanceof DirectoryReader) {
+ this.dir = ((DirectoryReader)reader).directory();
this.dirImpl = dir.getClass().getName();
- } catch (UnsupportedOperationException uoe) {
- this.dir = null;
- this.dirImpl = "N/A";
+ this.version = Long.toString(((DirectoryReader)reader).getVersion());
}
+ this.indexPath = indexPath;
+ lastModified = "N/A";
+ totalFileSize = dir == null ? -1 : Util.calcTotalFileSize(indexPath, dir);
+ fieldNames = new ArrayList<String>();
- this.indexCodec = IndexGate.getCodecOfFirstSegment(this.dir);
+ AtomicReader r;
+ if (reader instanceof CompositeReader) {
+ r = new SlowCompositeReaderWrapper((CompositeReader)reader);
+ } else {
+ r = (AtomicReader)reader;
+ }
- try {
- this.version = Long.toString(reader.getVersion());
- } catch (UnsupportedOperationException uoe) {
- this.dir = null;
+ FieldInfos infos = r.getFieldInfos();
+ for (FieldInfo info : infos) {
+ fieldNames.add(info.name);
}
- this.indexPath = indexPath;
- lastModified = dir == null ? "N/A" : new Date(IndexReader.lastModified(reader.directory())).toString();
- totalFileSize = dir == null ? -1 : Util.calcTotalFileSize(indexPath, reader.directory());
- fieldNames = new ArrayList<String>();
- fieldNames.addAll(reader.getFieldNames(FieldOption.ALL));
+
Collections.sort(fieldNames);
+ if (dir != null) {
+ indexFormat = IndexGate.getIndexFormat(dir);
+ }
}
private void countTerms() throws Exception {
termCounts = new HashMap<String,FieldTermCount>();
numTerms = 0;
Fields fields = MultiFields.getFields(reader);
- FieldsEnum fe = fields.iterator();
- String fld = null;
- while ((fld = fe.next()) != null) {
+
+ // if there are no postings, throw an exception
+ if (fields == null) {
+ throw new Exception("There are no postings in the index reader.");
+ }
+
+ Iterator<String> fe = fields.iterator();
+ TermsEnum te = null;
+
+ while (fe.hasNext()) {
+ String fld = fe.next();
FieldTermCount ftc = new FieldTermCount();
ftc.fieldname = fld;
- TermsEnum te = fe.terms();
- while (te.next() != null) {
- ftc.termCount++;
- numTerms++;
+ Terms terms = fields.terms(fld);
+ if (terms != null) {
+ te = terms.iterator(te);
+ while (te.next() != null) {
+ ftc.termCount++;
+ numTerms++;
+ }
}
termCounts.put(fld, ftc);
}
@@ -181,5 +204,9 @@ public class IndexInfo {
public IndexSearcher getIndexSearcher() {
return indexSearcher;
}
+
+ public int getIndexFormat() {
+ return indexFormat;
+ }
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/KeepAllIndexDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/KeepAllIndexDeletionPolicy.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/KeepAllIndexDeletionPolicy.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/KeepAllIndexDeletionPolicy.java Sun Jul 14 15:55:32 2013
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
-public class KeepAllIndexDeletionPolicy implements IndexDeletionPolicy {
+public class KeepAllIndexDeletionPolicy extends IndexDeletionPolicy {
public void onCommit(List<? extends IndexCommit> commits) throws IOException {
// do nothing - keep all points
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/KeepLastIndexDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/KeepLastIndexDeletionPolicy.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/KeepLastIndexDeletionPolicy.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/KeepLastIndexDeletionPolicy.java Sun Jul 14 15:55:32 2013
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
-public class KeepLastIndexDeletionPolicy implements IndexDeletionPolicy {
+public class KeepLastIndexDeletionPolicy extends IndexDeletionPolicy {
/**
* Deletes all commits except the most recent one.
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/NoScoringScorer.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/NoScoringScorer.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/NoScoringScorer.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/NoScoringScorer.java Sun Jul 14 15:55:32 2013
@@ -6,29 +6,39 @@ import org.apache.lucene.search.Scorer;
public class NoScoringScorer extends Scorer {
public static final NoScoringScorer INSTANCE = new NoScoringScorer();
-
+
protected NoScoringScorer() {
super(null);
}
-
+
@Override
public float score() throws IOException {
return 1.0f;
}
-
+
@Override
public int advance(int doc) throws IOException {
return 0;
}
-
+
@Override
public int docID() {
return 0;
}
-
+
@Override
public int nextDoc() throws IOException {
return 0;
}
-}
+ @Override
+ public int freq() throws IOException {
+ return 1;
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+
+}
\ No newline at end of file
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/TermStats.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/TermStats.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/TermStats.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/TermStats.java Sun Jul 14 15:55:32 2013
@@ -26,13 +26,13 @@ public final class TermStats {
public long totalTermFreq;
TermStats(String field, BytesRef termtext, int df) {
- this.termtext = new BytesRef(termtext);
+ this.termtext = (BytesRef)termtext.clone();
this.field = field;
this.docFreq = df;
}
TermStats(String field, BytesRef termtext, int df, long tf) {
- this.termtext = new BytesRef(termtext);
+ this.termtext = (BytesRef)termtext.clone();
this.field = field;
this.docFreq = df;
this.totalTermFreq = tf;
@@ -41,4 +41,8 @@ public final class TermStats {
String getTermText() {
return termtext.utf8ToString();
}
+
+ public String toString() {
+ return field + ":" + termtext.utf8ToString() + ":" + docFreq + ":" + totalTermFreq;
+ }
}
\ No newline at end of file
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/Util.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/Util.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/Util.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/Util.java Sun Jul 14 15:55:32 2013
@@ -21,11 +21,13 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.util.HashMap;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.DateTools.Resolution;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.util.BytesRef;
public class Util {
@@ -56,6 +58,10 @@ public class Util {
}
return sb.toString();
}
+
+ public static String bytesToHex(BytesRef bytes, boolean wrap) {
+ return bytesToHex(bytes.bytes, bytes.offset, bytes.length, wrap);
+ }
public static String bytesToHex(byte bytes[], int offset, int length, boolean wrap) {
StringBuffer sb = new StringBuffer();
@@ -155,31 +161,48 @@ public class Util {
return sb.toString();
}
- public static String fieldFlags(Field f) {
+ public static String fieldFlags(IndexableField f) {
if (f == null) {
return "-----------";
}
StringBuffer flags = new StringBuffer();
- if (f != null && f.isIndexed()) flags.append("I");
- else flags.append("-");
- if (f != null && f.isTokenized()) flags.append("T");
+ if (f != null && f.fieldType().indexed()) flags.append("I");
else flags.append("-");
- if (f != null && f.isStored()) flags.append("S");
+ if (f != null && f.fieldType().tokenized()) flags.append("T");
else flags.append("-");
- if (f != null && f.isTermVectorStored()) flags.append("V");
+ if (f != null && f.fieldType().stored()) flags.append("S");
else flags.append("-");
- if (f != null && f.isStoreOffsetWithTermVector()) flags.append("o");
+ if (f != null && f.fieldType().storeTermVectors()) flags.append("V");
else flags.append("-");
- if (f != null && f.isStorePositionWithTermVector()) flags.append("p");
+ if (f != null && f.fieldType().storeTermVectorOffsets()) flags.append("o");
else flags.append("-");
- if (f != null && f.getOmitTermFreqAndPositions()) flags.append("f");
+ if (f != null && f.fieldType().storeTermVectorPositions()) flags.append("p");
else flags.append("-");
- if (f != null && f.getOmitNorms()) flags.append("O");
+ if (f != null && f.fieldType().storeTermVectorPayloads()) flags.append("a");
else flags.append("-");
- if (f != null && f.isLazy()) flags.append("L");
- else flags.append("-");
- if (f != null && f.isBinary()) flags.append("B");
+ IndexOptions opts = f.fieldType().indexOptions();
+ // TODO: how to handle these codes
+ if (f.fieldType().indexed() && opts != null) {
+ switch (opts) {
+ case DOCS_ONLY:
+ flags.append("1");
+ break;
+ case DOCS_AND_FREQS:
+ flags.append("2");
+ break;
+ case DOCS_AND_FREQS_AND_POSITIONS:
+ flags.append("3");
+ break;
+ case DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS:
+ flags.append("4");
+ }
+ } else {
+ flags.append("-");
+ }
+ if (f != null && f.fieldType().omitNorms()) flags.append("O");
else flags.append("-");
+
+
return flags.toString();
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/BinaryDecoder.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/BinaryDecoder.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/BinaryDecoder.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/BinaryDecoder.java Sun Jul 14 15:55:32 2013
@@ -17,7 +17,7 @@ package org.apache.lucene.luke.core.deco
* limitations under the License.
*/
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
import org.apache.lucene.luke.core.Util;
public class BinaryDecoder implements Decoder {
@@ -34,7 +34,7 @@ public class BinaryDecoder implements De
}
@Override
- public String decodeStored(String fieldName, Fieldable value) throws Exception {
+ public String decodeStored(String fieldName, Field value) throws Exception {
return decodeTerm(fieldName, value);
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/DateDecoder.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/DateDecoder.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/DateDecoder.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/DateDecoder.java Sun Jul 14 15:55:32 2013
@@ -18,7 +18,7 @@ package org.apache.lucene.luke.core.deco
*/
import org.apache.lucene.document.DateTools;
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
public class DateDecoder implements Decoder {
@@ -28,7 +28,7 @@ public class DateDecoder implements Deco
}
@Override
- public String decodeStored(String fieldName, Fieldable value) throws Exception {
+ public String decodeStored(String fieldName, Field value) throws Exception {
return decodeTerm(fieldName, value.stringValue());
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/Decoder.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/Decoder.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/Decoder.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/Decoder.java Sun Jul 14 15:55:32 2013
@@ -17,10 +17,10 @@ package org.apache.lucene.luke.core.deco
* limitations under the License.
*/
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
public interface Decoder {
public String decodeTerm(String fieldName, Object value) throws Exception;
- public String decodeStored(String fieldName, Fieldable value) throws Exception;
+ public String decodeStored(String fieldName, Field value) throws Exception;
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/NumIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/NumIntDecoder.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/NumIntDecoder.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/NumIntDecoder.java Sun Jul 14 15:55:32 2013
@@ -17,7 +17,7 @@ package org.apache.lucene.luke.core.deco
* limitations under the License.
*/
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
@@ -30,7 +30,7 @@ public class NumIntDecoder implements De
}
@Override
- public String decodeStored(String fieldName, Fieldable value) {
+ public String decodeStored(String fieldName, Field value) {
return value.stringValue();
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/NumLongDecoder.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/NumLongDecoder.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/NumLongDecoder.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/NumLongDecoder.java Sun Jul 14 15:55:32 2013
@@ -17,7 +17,7 @@ package org.apache.lucene.luke.core.deco
* limitations under the License.
*/
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
@@ -30,7 +30,7 @@ public class NumLongDecoder implements D
}
@Override
- public String decodeStored(String fieldName, Fieldable value) {
+ public String decodeStored(String fieldName, Field value) {
return value.stringValue();
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/SolrDecoder.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/SolrDecoder.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/SolrDecoder.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/SolrDecoder.java Sun Jul 14 15:55:32 2013
@@ -22,7 +22,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.TreeMap;
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
import org.apache.lucene.luke.core.ClassFinder;
import org.apache.solr.schema.FieldType;
@@ -90,7 +90,7 @@ public class SolrDecoder implements Deco
return fieldType.indexedToReadable(value.toString());
}
- public String decodeStored(String fieldName, Fieldable value)
+ public String decodeStored(String fieldName, Field value)
throws Exception {
return fieldType.storedToReadable(value);
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/StringDecoder.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/StringDecoder.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/StringDecoder.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/core/decoders/StringDecoder.java Sun Jul 14 15:55:32 2013
@@ -17,7 +17,7 @@ package org.apache.lucene.luke.core.deco
* limitations under the License.
*/
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
public class StringDecoder implements Decoder {
@@ -27,7 +27,7 @@ public class StringDecoder implements De
}
@Override
- public String decodeStored(String fieldName, Fieldable value) {
+ public String decodeStored(String fieldName, Field value) {
return decodeTerm(fieldName, value.stringValue());
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/ui/AnalyzersTab.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/ui/AnalyzersTab.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/ui/AnalyzersTab.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/ui/AnalyzersTab.java Sun Jul 14 15:55:32 2013
@@ -13,11 +13,11 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.index.Payload;
import org.apache.lucene.luke.core.Util;
import org.apache.lucene.luke.ui.LukeWindow.LukeMediator;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.apache.pivot.beans.BXML;
import org.apache.pivot.beans.Bindable;
@@ -201,9 +201,9 @@ public class AnalyzersTab extends TableP
OffsetAttribute off = (OffsetAttribute) att;
val = off.startOffset() + "," + off.endOffset();
} else if (attClass.equals("PayloadAttribute")) {
- Payload payload = ((PayloadAttribute) att).getPayload();
+ BytesRef payload = ((PayloadAttribute) att).getPayload();
if (payload != null) {
- byte[] data = payload.getData();
+ byte[] data = payload.bytes;
val = Util.bytesToHex(data, 0, data.length, false);
} else {
val = "";
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/ui/DocumentsTab.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/ui/DocumentsTab.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/ui/DocumentsTab.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/ui/DocumentsTab.java Sun Jul 14 15:55:32 2013
@@ -22,10 +22,17 @@ import java.net.URL;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.CompositeReader;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -36,7 +43,6 @@ import org.apache.lucene.luke.core.decod
import org.apache.lucene.luke.ui.LukeWindow.LukeMediator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Similarity;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.pivot.beans.BXML;
@@ -98,8 +104,10 @@ public class DocumentsTab extends TableP
private String fld;
private Term lastTerm;
private IndexReader ir;
+ private AtomicReader ar;
private java.util.List<String> indexFields;
private IndexInfo idxInfo;
+ private FieldInfos infos;
public DocumentsTab() {
Action.getNamedActions().put("nextDoc", new Action() {
@@ -183,6 +191,17 @@ public class DocumentsTab extends TableP
public void onOpenIndex() throws Exception {
this.idxInfo = lukeMediator.getIndexInfo();
this.ir = idxInfo.getReader();
+
+ if (ir instanceof CompositeReader) {
+ ar = new SlowCompositeReaderWrapper((CompositeReader)ir);
+ } else if (ir instanceof AtomicReader) {
+ ar = (AtomicReader)ir;
+ }
+
+ if (ar != null) {
+ infos = ar.getFieldInfos();
+ }
+
this.indexFields = idxInfo.getFieldNames();
@@ -191,11 +210,13 @@ public class DocumentsTab extends TableP
fields.add(s);
}
fieldsList.setListData(fields);
- fieldsList.setSelectedIndex(0);
+
fieldNames = idxInfo.getFieldNames();
if (fieldNames.size() == 0) {
// TODO:
// showStatus("Empty index.");
+ } else {
+ fieldsList.setSelectedIndex(0);
}
maxDocs.setText(String.valueOf(ir.maxDoc() - 1));
}
@@ -220,8 +241,9 @@ public class DocumentsTab extends TableP
return;
}
docNum.setText(String.valueOf(iNum));
- org.apache.lucene.util.Bits deleted = MultiFields.getDeletedDocs(ir);
- if (deleted == null || !deleted.get(iNum)) {
+
+ org.apache.lucene.util.Bits live = ar.getLiveDocs();
+ if (live == null || live.get(iNum)) {
Task<Object> populateTableTask = new Task<Object>() {
@Override
@@ -289,7 +311,7 @@ public class DocumentsTab extends TableP
for (int i = 0; i < indexFields.size(); i++) {
Map<String,String> row = new HashMap<String,String>();
- Field[] fields = doc.getFields(indexFields.get(i));
+ IndexableField[] fields = doc.getFields(indexFields.get(i));
if (fields == null) {
tableData.add(row);
addFieldRow(row, indexFields.get(i), null, docid);
@@ -306,10 +328,12 @@ public class DocumentsTab extends TableP
}
}
- private void addFieldRow(Map<String,String> row, String fName, Field f,
+ private void addFieldRow(Map<String,String> row, String fName, IndexableField field,
int docid) {
java.util.Map<String,Decoder> decoders = lukeMediator.getDecoders();
Decoder defDecoder = lukeMediator.getDefDecoder();
+
+ Field f = (Field) field;
// putProperty(row, "field", f);
// putProperty(row, "fName", fName);
@@ -322,9 +346,11 @@ public class DocumentsTab extends TableP
if (f != null) {
try {
- if (ir.hasNorms(fName)) {
- row.put("norm",
- String.valueOf(Similarity.decodeNorm(ir.norms(fName)[docid])));
+ FieldInfo info = infos.fieldInfo(fName);
+ if (info.hasNorms()) {
+ NumericDocValues norms = ar.getNormValues(fName);
+ String val = Long.toString(norms.get(docid));
+ row.put("norm", String.valueOf(norms.get(docid)));
} else {
row.put("norm", "---");
}
@@ -339,14 +365,17 @@ public class DocumentsTab extends TableP
if (f != null) {
String text = f.stringValue();
- if (text == null && f.isBinary()) {
- text = Util.bytesToHex(f.getBinaryValue(), f.getBinaryOffset(),
- f.getBinaryLength(), false);
+ if (text == null) {
+ if (f.binaryValue() != null) {
+ text = Util.bytesToHex(f.binaryValue(), false);
+ } else {
+ text = "(null)";
+ }
}
Decoder dec = decoders.get(f.name());
if (dec == null) dec = defDecoder;
try {
- if (f.isStored()) {
+ if (f.fieldType().stored()) {
text = dec.decodeStored(f.name(), f);
} else {
text = dec.decodeTerm(f.name(), text);
@@ -377,7 +406,7 @@ public class DocumentsTab extends TableP
fld = (String) fieldsList.getSelectedItem();
System.out.println("fld:" + fld);
Terms terms = MultiFields.getTerms(ir, fld);
- te = terms.iterator();
+ te = terms.iterator(null);
BytesRef term = te.next();
showTerm(new Term(fld, term));
} catch (Exception e) {
@@ -643,10 +672,10 @@ public class DocumentsTab extends TableP
if (te == null || !DocumentsTab.this.fld.equals(fld)
|| !text.equals(rawString)) {
Terms terms = MultiFields.getTerms(ir, fld);
- te = terms.iterator();
+ te = terms.iterator(null);
DocumentsTab.this.fld = fld;
- status = te.seek(new BytesRef(text));
+ status = te.seekCeil(new BytesRef(text));
if (status.equals(SeekStatus.FOUND)) {
rawTerm = te.term();
} else {
@@ -665,7 +694,7 @@ public class DocumentsTab extends TableP
if (terms == null) {
continue;
}
- te = terms.iterator();
+ te = terms.iterator(null);
rawTerm = te.next();
DocumentsTab.this.fld = fld;
break;
@@ -795,10 +824,6 @@ public class DocumentsTab extends TableP
e.printStackTrace();
// TODO:
//errorMsg(e.getMessage());
- } finally {
- if (is != null) try {
- is.close();
- } catch (Exception e1) {}
}
return null;
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/ui/FilesTab.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/ui/FilesTab.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/ui/FilesTab.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/ui/FilesTab.java Sun Jul 14 15:55:32 2013
@@ -26,6 +26,7 @@ import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexGate;
import org.apache.lucene.index.IndexReader;
@@ -133,7 +134,7 @@ public class FilesTab extends SplitPane
System.out.println("not available");
return;
}
- Collection<IndexCommit> commits = IndexReader.listCommits(dir);
+ java.util.List<IndexCommit> commits = DirectoryReader.listCommits(dir);
// commits are ordered from oldest to newest ?
Iterator<IndexCommit> it = commits.iterator();
int rowNum = 0;
@@ -157,10 +158,10 @@ public class FilesTab extends SplitPane
char[] flags = new char[] {'-', '-'};
if (commit.isDeleted()) flags[0] = 'D';
- if (commit.isOptimized()) flags[1] = 'O';
+
+ // TODO: show segment count instead?
+ //if (commit.isOptimized()) flags[1] = 'O';
row.put("flags", new String(flags));
- row.put("version", Long.toHexString(commit.getVersion()));
- row.put("timestamp", new Date(commit.getTimestamp()).toString());
java.util.Map<String,String> userData = commit.getUserData();
if (userData != null && !userData.isEmpty()) {
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/ui/LukeWindow.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/ui/LukeWindow.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/ui/LukeWindow.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/ui/LukeWindow.java Sun Jul 14 15:55:32 2013
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.URL;
import java.util.Arrays;
-import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -31,17 +30,15 @@ import org.apache.lucene.analysis.core.S
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.luke.core.ClassFinder;
import org.apache.lucene.luke.core.IndexInfo;
-import org.apache.lucene.luke.core.KeepAllIndexDeletionPolicy;
-import org.apache.lucene.luke.core.KeepLastIndexDeletionPolicy;
-import org.apache.lucene.luke.core.Prefs;
import org.apache.lucene.luke.core.decoders.Decoder;
import org.apache.lucene.luke.core.decoders.StringDecoder;
import org.apache.lucene.store.Directory;
@@ -50,7 +47,6 @@ import org.apache.lucene.store.RAMDirect
import org.apache.lucene.util.Version;
import org.apache.pivot.beans.BXML;
import org.apache.pivot.beans.Bindable;
-import org.apache.pivot.collections.ArrayList;
import org.apache.pivot.collections.Map;
import org.apache.pivot.util.Resources;
import org.apache.pivot.wtk.Alert;
@@ -178,9 +174,10 @@ public class LukeWindow extends Frame im
}
boolean existsSingle = false;
try {
- existsSingle = IndexReader.indexExists(d);
- } catch (Exception e) {
- //
+ new SegmentInfos().read(d);
+ existsSingle = true;
+ } catch (Throwable e) {
+ e.printStackTrace();
}
if (!existsSingle) { // try multi
@@ -208,9 +205,11 @@ public class LukeWindow extends Frame im
}
existsSingle = false;
try {
- existsSingle = IndexReader.indexExists(d1);
- } catch (Exception e) {}
- ;
+ new SegmentInfos().read(d1);
+ existsSingle = true;
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
if (!existsSingle) {
d1.close();
continue;
@@ -239,37 +238,33 @@ public class LukeWindow extends Frame im
directory.close();
directory = dir1;
}
- IndexDeletionPolicy policy;
- if (keepCommits) {
- policy = new KeepAllIndexDeletionPolicy();
- } else {
- policy = new KeepLastIndexDeletionPolicy();
- }
java.util.ArrayList<IndexReader> readers = new java.util.ArrayList<IndexReader>();
for (Directory dd : dirs) {
IndexReader reader;
if (tiiDivisor > 1) {
- reader = IndexReader.open(dd, policy, readOnly,
- tiiDivisor);
+ reader = IndexReader.open(dd, tiiDivisor);
} else {
- reader = IndexReader.open(dd, policy, readOnly);
+ reader = IndexReader.open(dd);
}
readers.add(reader);
}
if (readers.size() == 1) {
indexReader = readers.get(0);
- directory = indexReader.directory();
+ IndexReader ir = readers.get(0);
+ directory = ((DirectoryReader)ir).directory();
} else {
indexReader = new MultiReader(
(IndexReader[]) readers.toArray(new IndexReader[readers.size()]));
}
- lukeMediator.indexInfo = new IndexInfo(indexReader, this.indexPath, readOnly);
+ lukeMediator.indexInfo = new IndexInfo(indexReader, this.indexPath);
- Collection<String> fieldNames = indexReader.getFieldNames(IndexReader.FieldOption.ALL);
- if (fieldNames.size() == 0) {
- // showStatus("Empty index.");
- }
+ // TODO:
+// Collection<String> fieldNames = indexReader.getFieldNames(IndexReader.FieldOption.ALL);
+//
+// if (fieldNames.size() == 0) {
+// // showStatus("Empty index.");
+// }
// call onOpenIndex for all tabs
overviewTab.onOpenIndex();
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/ui/OverviewTab.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/ui/OverviewTab.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/ui/OverviewTab.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/ui/OverviewTab.java Sun Jul 14 15:55:32 2013
@@ -21,11 +21,13 @@ import java.io.IOException;
import java.net.URL;
import java.text.NumberFormat;
import java.util.Collections;
-import java.util.Date;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexGate;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.luke.core.FieldTermCount;
import org.apache.lucene.luke.core.HighFreqTerms;
import org.apache.lucene.luke.core.IndexInfo;
@@ -42,7 +44,6 @@ import org.apache.pivot.collections.List
import org.apache.pivot.collections.Map;
import org.apache.pivot.collections.Sequence;
import org.apache.pivot.serialization.SerializationException;
-import org.apache.pivot.util.Filter;
import org.apache.pivot.util.Resources;
import org.apache.pivot.util.concurrent.Task;
import org.apache.pivot.util.concurrent.TaskExecutionException;
@@ -58,13 +59,6 @@ import org.apache.pivot.wtk.Spinner;
import org.apache.pivot.wtk.SplitPane;
import org.apache.pivot.wtk.TablePane;
import org.apache.pivot.wtk.TableView;
-import org.apache.pivot.wtk.TableView.CellRenderer;
-import org.apache.pivot.wtk.TableView.Column;
-import org.apache.pivot.wtk.TableView.HeaderDataRenderer;
-import org.apache.pivot.wtk.TableView.RowEditor;
-import org.apache.pivot.wtk.TableView.SelectMode;
-import org.apache.pivot.wtk.TableViewColumnListener;
-import org.apache.pivot.wtk.TableViewListener;
import org.apache.pivot.wtk.TableViewSortListener;
import org.apache.pivot.wtk.TaskAdapter;
@@ -196,15 +190,15 @@ public class OverviewTab extends SplitPa
}
// we need IndexReader from now on
- // TODO: indexes that may have different codecs for different segments??
- indexCodec.setText(lukeMediator.getIndexInfo().getIndexCodec().name);
+ // TODO: Codecs are now per field...
+ // indexCodec.setText(lukeMediator.getIndexInfo().getIndexCodec().getName());
- String modText = "N/A";
- if (dir != null) {
- modText = new Date(IndexReader.lastModified(dir))
- .toString();
- }
- iMod.setText(modText);
+// String modText = "N/A";
+// if (dir != null) {
+// modText = new Date(IndexReader.lastModified(dir))
+// .toString();
+// }
+// iMod.setText(modText);
String numdocs = String.valueOf(ir.numDocs());
iDocs.setText(numdocs);
@@ -295,13 +289,15 @@ public class OverviewTab extends SplitPa
String sDel = ir.hasDeletions() ? "Yes (" + ir.numDeletedDocs() + ")"
: "No";
- String sDelOpt = sDel + " / " + (ir.isOptimized() ? "Yes" : "No");
+ IndexCommit ic = ir instanceof DirectoryReader ? ((DirectoryReader)ir).getIndexCommit() : null;
+ String sDelOpt = ic != null ? (ic.getSegmentCount() == 1 ? "Yes" : "No") : "?";
iDelOpt.setText(sDelOpt);
+
+
String verText = "N/A";
- if (dir != null) {
- verText = Long.toHexString(IndexReader
- .getCurrentVersion(dir));
+ if (ic != null) {
+ verText = Long.toHexString(((DirectoryReader)ir).getVersion());
}
iVer.setText(verText);
@@ -322,24 +318,30 @@ public class OverviewTab extends SplitPa
String divText = "N/A";
- try {
- divText = String.valueOf(ir.getTermInfosIndexDivisor());
- } catch (UnsupportedOperationException uoe) {}
+ if (ir instanceof DirectoryReader) {
+ java.util.List<AtomicReaderContext> readers = ((DirectoryReader) ir).leaves();
+ if (readers.size() > 0)
+ {
+ if (readers.get(0).reader() instanceof SegmentReader) {
+ divText = String.valueOf(((SegmentReader) readers.get(0).reader()).getTermInfosIndexDivisor());
+ }
+ }
+ }
iTiiDiv.setText(divText);
String commitText = "N/A";
try {
- IndexCommit commit = ir.getIndexCommit();
- commitText = commit.getSegmentsFileName() + " ("
- + new Date(commit.getTimestamp()).toString() + ")";
+ commitText = ic.getSegmentsFileName() +
+ " (generation=" + ic.getGeneration() + ", segs=" +
+ ic.getSegmentCount() + ")";
} catch (UnsupportedOperationException uoe) {}
iCommit.setText(commitText);
String userData = null;
try {
- java.util.Map<String,String> userDataMap = ir.getCommitUserData();
+ java.util.Map<String,String> userDataMap = ic.getUserData();
if (userDataMap != null && !userDataMap.isEmpty()) {
- userData = ir.getCommitUserData().toString();
+ userData = ic.getUserData().toString();
} else {
userData = "--";
}
Modified: lucene/sandbox/luke/src/org/apache/lucene/luke/ui/SearchTab.java
URL: http://svn.apache.org/viewvc/lucene/sandbox/luke/src/org/apache/lucene/luke/ui/SearchTab.java?rev=1502987&r1=1502986&r2=1502987&view=diff
==============================================================================
--- lucene/sandbox/luke/src/org/apache/lucene/luke/ui/SearchTab.java (original)
+++ lucene/sandbox/luke/src/org/apache/lucene/luke/ui/SearchTab.java Sun Jul 14 15:55:32 2013
@@ -26,7 +26,8 @@ import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.luke.core.AccessibleHitCollector;
import org.apache.lucene.luke.core.AccessibleTopHitCollector;
import org.apache.lucene.luke.core.IndexInfo;
@@ -35,12 +36,11 @@ import org.apache.lucene.luke.core.Prefs
import org.apache.lucene.luke.core.Util;
import org.apache.lucene.luke.core.decoders.Decoder;
import org.apache.lucene.luke.ui.LukeWindow.LukeMediator;
-import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.util.Version;
import org.apache.pivot.beans.BXML;
import org.apache.pivot.beans.Bindable;
@@ -170,7 +170,7 @@ public class SearchTab extends SplitPane
}
Object srchOpts = null; //find("srchOptTabs");
// query parser opts
- Similarity sim = createSimilarity();
+ DefaultSimilarity sim = createSimilarity();
AccessibleHitCollector col;
try {
col = createCollector();
@@ -427,7 +427,7 @@ public class SearchTab extends SplitPane
return res;
}
- public Similarity createSimilarity() {
+ public DefaultSimilarity createSimilarity() {
// Object ckSimDef = find(srchOpts, "ckSimDef");
// Object ckSimSweet = find(srchOpts, "ckSimSweet");
// Object ckSimOther = find(srchOpts, "ckSimOther");
@@ -571,14 +571,14 @@ public class SearchTab extends SplitPane
String fieldName = fields.get(j);
Decoder dec = lukeMediator.getDecoders().get(fieldName);
if (dec == null) dec = lukeMediator.getDefDecoder();
- Fieldable[] values = doc.getFieldables(fieldName);
+ IndexableField[] values = doc.getFields(fieldName);
vals.setLength(0);
boolean decodeErr = false;
if (values != null) for (int k = 0; k < values.length; k++) {
if (k > 0) vals.append(' ');
String v;
try {
- v = dec.decodeStored(fieldName, values[k]);
+ v = dec.decodeStored(fieldName, (Field) values[k]);
} catch (Throwable e) {
e.printStackTrace();
v = values[k].stringValue();