You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ha...@apache.org on 2013/09/01 18:32:18 UTC
svn commit: r1519288 - in /lucene/dev/branches/lucene3069/lucene:
codecs/src/java/org/apache/lucene/codecs/sep/
codecs/src/java/org/apache/lucene/codecs/temp/
core/src/java/org/apache/lucene/codecs/lucene41/
Author: han
Date: Sun Sep 1 16:32:17 2013
New Revision: 1519288
URL: http://svn.apache.org/r1519288
Log:
add javadocs, pass precommit
Added:
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/package.html (with props)
Modified:
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsWriter.java
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdPostingsFormat.java
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsReader.java
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsWriter.java
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java
lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java
lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java
lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java
lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipWriter.java
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsWriter.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsWriter.java Sun Sep 1 16:32:17 2013
@@ -164,7 +164,7 @@ public final class SepPostingsWriter ext
}
@Override
- public SepTermState newTermState() {
+ public BlockTermState newTermState() {
return new SepTermState();
}
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdPostingsFormat.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdPostingsFormat.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdPostingsFormat.java Sun Sep 1 16:32:17 2013
@@ -32,6 +32,22 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
+/**
+ * FST-based term dict, using ord as FST output.
+ *
+ * The FST holds the mapping between <term, ord>, and
+ * term's metadata is delta encoded into a single byte block.
+ *
+ * Typically the byte block consists of four parts:
+ * 1. term statistics: docFreq, totalTermFreq;
+ * 2. monotonic long[], e.g. the pointer to the postings list for that term;
+ * 3. generic byte[], e.g. other information customized by postings base.
+ * 4. single-level skip list to speed up metadata decoding by ord.
+ *
+ * <!-- TODO: explain about the data format -->
+ * @lucene.experimental
+ */
+
public final class TempFSTOrdPostingsFormat extends PostingsFormat {
public TempFSTOrdPostingsFormat() {
super("TempFSTOrd");
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsReader.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsReader.java Sun Sep 1 16:32:17 2013
@@ -61,6 +61,15 @@ import org.apache.lucene.codecs.FieldsPr
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.CodecUtil;
+/**
+ * FST-based terms dictionary reader.
+ *
+ * The FST index maps each term and its ord, and during seek
+ * the ord is used fetch metadata from a single block.
+ * The term dictionary is fully memeory resident.
+ *
+ * @lucene.experimental
+ */
public class TempFSTOrdTermsReader extends FieldsProducer {
static final int INTERVAL = TempFSTOrdTermsWriter.SKIP_INTERVAL;
final TreeMap<String, TermsReader> fields = new TreeMap<String, TermsReader>();
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsWriter.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsWriter.java Sun Sep 1 16:32:17 2013
@@ -45,8 +45,11 @@ import org.apache.lucene.codecs.TermsCon
import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.codecs.CodecUtil;
-/** FST based term dict, only ords is hold in FST,
- * other metadata encoded into single byte block */
+/**
+ * FST based term dict, the FST maps each term and its ord.
+ *
+ * @lucene.experimental
+ */
public class TempFSTOrdTermsWriter extends FieldsConsumer {
static final String TERMS_INDEX_EXTENSION = "tix";
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java Sun Sep 1 16:32:17 2013
@@ -32,6 +32,21 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
+/**
+ * FST-based term dict, using metadata as FST output.
+ *
+ * The FST directly holds the mapping between <term, metadata>.
+ *
+ * Term metadata consists of three parts:
+ * 1. term statistics: docFreq, totalTermFreq;
+ * 2. monotonic long[], e.g. the pointer to the postings list for that term;
+ * 3. generic byte[], e.g. other information need by postings reader.
+ *
+ *
+ * <!-- TODO: explain about the data format -->
+ * @lucene.experimental
+ */
+
public final class TempFSTPostingsFormat extends PostingsFormat {
public TempFSTPostingsFormat() {
super("TempFST");
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java Sun Sep 1 16:32:17 2013
@@ -59,6 +59,15 @@ import org.apache.lucene.codecs.FieldsPr
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.CodecUtil;
+/**
+ * FST-based terms dictionary reader.
+ *
+ * The FST directly maps each term and its metadata,
+ * it is memeory resident.
+ *
+ * @lucene.experimental
+ */
+
public class TempFSTTermsReader extends FieldsProducer {
final TreeMap<String, TermsReader> fields = new TreeMap<String, TermsReader>();
final PostingsReaderBase postingsReader;
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java Sun Sep 1 16:32:17 2013
@@ -44,8 +44,11 @@ import org.apache.lucene.codecs.TermsCon
import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.codecs.CodecUtil;
-/** FST based term dict, all the metadata held
- * as output of FST */
+/**
+ * FST based term dict, the FST maps each term and its metadata.
+ *
+ * @lucene.experimental
+ */
public class TempFSTTermsWriter extends FieldsConsumer {
static final String TERMS_EXTENSION = "tmp";
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java Sun Sep 1 16:32:17 2013
@@ -27,14 +27,26 @@ import org.apache.lucene.store.DataOutpu
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.LongsRef;
+/**
+ * An FST {@link Outputs} implementation for
+ * {@link TempFSTPostingsFormat}.
+ *
+ * @lucene.experimental
+ */
+
// NOTE: outputs should be per-field, since
// longsSize is fixed for each field
public class TempTermOutputs extends Outputs<TempTermOutputs.TempMetaData> {
private final static TempMetaData NO_OUTPUT = new TempMetaData();
- private static boolean DEBUG = false;
+ //private static boolean TEST = false;
private final boolean hasPos;
private final int longsSize;
+ /**
+ * Represents the metadata for one term.
+ * On an FST, only long[] part is 'shared' and pushed towards root.
+ * byte[] and term stats will be kept on deeper arcs.
+ */
public static class TempMetaData {
long[] longs;
byte[] bytes;
@@ -89,33 +101,6 @@ public class TempTermOutputs extends Out
bytesEqual(this, other);
}
- public String toString() {
- if (this == NO_OUTPUT) {
- return "no_output";
- }
- StringBuffer sb = new StringBuffer();
- if (longs != null) {
- sb.append("[ ");
- for (int i = 0; i < longs.length; i++) {
- sb.append(longs[i]+" ");
- }
- sb.append("]");
- } else {
- sb.append("null");
- }
- if (bytes != null) {
- sb.append(" [ ");
- for (int i = 0; i < bytes.length; i++) {
- sb.append(Integer.toHexString((int)bytes[i] & 0xff)+" ");
- }
- sb.append("]");
- } else {
- sb.append(" null");
- }
- sb.append(" "+docFreq);
- sb.append(" "+totalTermFreq);
- return sb.toString();
- }
}
protected TempTermOutputs(FieldInfo fieldInfo, int longsSize) {
@@ -130,14 +115,10 @@ public class TempTermOutputs extends Out
// 1. every value in t1 is not larger than in t2, or
// 2. every value in t1 is not smaller than t2.
//
- // NOTE:
- // Only long[] part is 'shared' and pushed towards root.
- // byte[] and term stats will be kept on deeper arcs.
- //
public TempMetaData common(TempMetaData t1, TempMetaData t2) {
- //if (DEBUG) System.out.print("common("+t1+", "+t2+") = ");
+ //if (TEST) System.out.print("common("+t1+", "+t2+") = ");
if (t1 == NO_OUTPUT || t2 == NO_OUTPUT) {
- //if (DEBUG) System.out.println("ret:"+NO_OUTPUT);
+ //if (TEST) System.out.println("ret:"+NO_OUTPUT);
return NO_OUTPUT;
}
assert t1.longs.length == t2.longs.length;
@@ -172,15 +153,15 @@ public class TempTermOutputs extends Out
ret = new TempMetaData(min, null, 0, -1);
}
}
- //if (DEBUG) System.out.println("ret:"+ret);
+ //if (TEST) System.out.println("ret:"+ret);
return ret;
}
@Override
public TempMetaData subtract(TempMetaData t1, TempMetaData t2) {
- //if (DEBUG) System.out.print("subtract("+t1+", "+t2+") = ");
+ //if (TEST) System.out.print("subtract("+t1+", "+t2+") = ");
if (t2 == NO_OUTPUT) {
- //if (DEBUG) System.out.println("ret:"+t1);
+ //if (TEST) System.out.println("ret:"+t1);
return t1;
}
assert t1.longs.length == t2.longs.length;
@@ -201,7 +182,7 @@ public class TempTermOutputs extends Out
} else {
ret = new TempMetaData(share, t1.bytes, t1.docFreq, t1.totalTermFreq);
}
- //if (DEBUG) System.out.println("ret:"+ret);
+ //if (TEST) System.out.println("ret:"+ret);
return ret;
}
@@ -210,12 +191,12 @@ public class TempTermOutputs extends Out
// we seem to put much stress on FST Outputs decoding?
@Override
public TempMetaData add(TempMetaData t1, TempMetaData t2) {
- //if (DEBUG) System.out.print("add("+t1+", "+t2+") = ");
+ //if (TEST) System.out.print("add("+t1+", "+t2+") = ");
if (t1 == NO_OUTPUT) {
- //if (DEBUG) System.out.println("ret:"+t2);
+ //if (TEST) System.out.println("ret:"+t2);
return t2;
} else if (t2 == NO_OUTPUT) {
- //if (DEBUG) System.out.println("ret:"+t1);
+ //if (TEST) System.out.println("ret:"+t1);
return t1;
}
assert t1.longs.length == t2.longs.length;
@@ -234,7 +215,7 @@ public class TempTermOutputs extends Out
} else {
ret = new TempMetaData(accum, t1.bytes, t1.docFreq, t1.totalTermFreq);
}
- //if (DEBUG) System.out.println("ret:"+ret);
+ //if (TEST) System.out.println("ret:"+ret);
return ret;
}
Added: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/package.html?rev=1519288&view=auto
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/package.html (added)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/package.html Sun Sep 1 16:32:17 2013
@@ -0,0 +1,25 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+FST term dict: FST-based term dictionary implementations.
+</body>
+</html>
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java Sun Sep 1 16:32:17 2013
@@ -33,7 +33,7 @@ import static org.apache.lucene.codecs.l
* Encode all values in normal area with fixed bit width,
* which is determined by the max value in this block.
*/
-public final class ForUtil {
+final class ForUtil {
/**
* Special number of bits per value used whenever all values to encode are equal.
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java Sun Sep 1 16:32:17 2013
@@ -575,7 +575,8 @@ public final class Lucene41PostingsWrite
}
lastState = state;
}
- public void _encodeTerm(DataOutput out, FieldInfo fieldInfo, IntBlockTermState state) throws IOException {
+
+ private void _encodeTerm(DataOutput out, FieldInfo fieldInfo, IntBlockTermState state) throws IOException {
if (state.singletonDocID == -1) {
out.writeVLong(state.docTermStartFP - lastState.docTermStartFP);
lastState.docTermStartFP = state.docTermStartFP;
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java Sun Sep 1 16:32:17 2013
@@ -50,7 +50,7 @@ import org.apache.lucene.store.IndexInpu
* Therefore, we'll trim df before passing it to the interface. see trim(int)
*
*/
-public final class Lucene41SkipReader extends MultiLevelSkipListReader {
+final class Lucene41SkipReader extends MultiLevelSkipListReader {
// private boolean DEBUG = Lucene41PostingsReader.DEBUG;
private final int blockSize;
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipWriter.java?rev=1519288&r1=1519287&r2=1519288&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipWriter.java Sun Sep 1 16:32:17 2013
@@ -43,7 +43,7 @@ import org.apache.lucene.codecs.MultiLev
* 4. start offset.
*
*/
-public final class Lucene41SkipWriter extends MultiLevelSkipListWriter {
+final class Lucene41SkipWriter extends MultiLevelSkipListWriter {
// private boolean DEBUG = Lucene41PostingsReader.DEBUG;
private int[] lastSkipDoc;